diff options
author | Christopher Vogt <oss.nsp@cvogt.org> | 2016-02-06 13:03:36 -0500 |
---|---|---|
committer | Christopher Vogt <oss.nsp@cvogt.org> | 2016-03-04 15:06:30 -0500 |
commit | 974942db43ff2d1fa7ba71ad60f9bb9eae2d8631 (patch) | |
tree | d7235df9d4d6a67753dc2a20ab6bfcb7a24dc74c | |
download | cbt-974942db43ff2d1fa7ba71ad60f9bb9eae2d8631.tar.gz cbt-974942db43ff2d1fa7ba71ad60f9bb9eae2d8631.tar.bz2 cbt-974942db43ff2d1fa7ba71ad60f9bb9eae2d8631.zip |
CBT Version 1.0-BETA
32 files changed, 2394 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1eee79b --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +config +cache +classes +lib +out +scala_classes +target/ +*.login +cache_ diff --git a/DEVELOPER_GUIDE.txt b/DEVELOPER_GUIDE.txt new file mode 100644 index 0000000..78b9963 --- /dev/null +++ b/DEVELOPER_GUIDE.txt @@ -0,0 +1,21 @@ +Welcome developer. + +CBT has a very easy code base that you can fully master in under an hour. +Don't shy away to submit PRs :). + +CBT launches in stages in order to bootstrap from source out of Java into Scala and finally CBT. + +The ./cbt bash script starts the process. + +You currently need javac, nailgun, gpg and realpath or gcc installed. + +CBT's directory structure + +cbt Shell script launching cbt. Can be symlinked. +bootstrap_scala/ Self-contained downloader for the core Scala jars. Allows bootstrapping from Java into Scala. +nailgun_launcher/ Self-contained helper that allows using Nailgun with minimal permanent classpath. (Is this actually needed?) +realpath/ Self-contained realpath source code to correctly figure our CBTs home directory. (Open for replacement ideas.) +stage1/ CBT's code that only relies only on Scala/Java built-ins. Contains a Maven resolver to download libs for stage2. +stage2/ CBT's code that requires additional libs, e.g. barbary watchservice. +test/ Unit tests that can serve as example builds +sonatype.login Sonatype credentials for deployment. Not in git obviously. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..e3ae0ac --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,25 @@ +Simplified BSD License + +Copyright (c) 2015, Jan Christopher Vogt +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..e0a8fb5 --- /dev/null +++ b/README.txt @@ -0,0 +1,124 @@ +Welcome to Compossible Build Tool (CBT) for Scala + +Fun, fast, intuitive, composable and statically checked builds written in Scala. + +Currently CBT has been tested in OSX only. Adding support for Unix and Windows +should not be hard. Please contribute back if you mange :). + +CBT supports the basic needs for Scala builds right now. +Composing, compiling, running, testing, packaging, publishing. +Tasks outside of these, such as building manuals will require +easy custom code. If you integrate something, consider +doing it as traits that you make available as a library that +other builds can depend on and mix in. + +Getting started: + +You currently need javac, nailgun, gpg and realpath or gcc installed. + +CBT bootstraps from source. To install, just clone the repository. +To use, just call the "cbt" bash script. You will see CBT first +building up itself, then trying to start your build. + +The easiest CBT build requires no script. It compiles source files from the +current directory and from "src/ into the target folder. Try calling "cbt compile". +If you have a class called Main with a main method, try "cbt run". +Put a source file with a Main class into "test/" in order for +"cbt test" to run it. It will see your main code. + +If you need more than this, like dependencies create a scala file in build/ +that describes your build. Here is an example + +// build/build.scala +class Build(context: cbt.Context) extends cbt.Build(context){ + override def version = "0.6.1-SNAPSHOT" + override def groupId = "org.cvogt" + override def artifactId = "play-json-extensions" + override def dependencies = Vector( + "com.typesafe.play" %% "play-json" % "2.4.4" + ) + override def compile = { + println("Compiling...") + super.compile + } + def foo = "Hello World" +} + +Now you can call methods of this class through cbt. Try "cbt foo". +You can see how your build is configured via overrides. + +call "cbt" to see a full list of available commands for this build. + +Look into the class DefaultBuild in CBT's source code to see their +details. The source code is really simple. Don't shy away from +looking, even as a beginner. No crazy stuff, I promise ;). You +can find the relevant code in CBT's stage2/DefaultBuild.scala + +I order to keep executing the same command triggered by file changes use "cbt loop <command>". + +You can find example builds in CBT's own "test/"" folder. +Not all of them have a build file, in which case CBT uses the default +cbt.DefaultBuild. + +A folder "build/" can have its own folder "build/" inside in order +to add source or maven dependencies to your build. Eventually +you'll be able to also choose the CBT and Scala versions for +target builds. Make sure you extend cbt.BuilBuild instead of +cbt.Build, in order to automatically trigger building of the +target build. + +cbt is fast. It uses Nailgun to keep the JVM hot. It uses the Java +WatchService (respectively a fast OSX port of it) for instant triggering +re-compilation on file changes. Use "cbt loop compile". + +CBT concepts + +There two essential primitives available in build scripts for composing +modular projects: + + 1 Dynamically compiling and loading Build scripts in other + directories and calling methods (aka tasks) on them to compile, + get the classpath, ask for version numbers, etc. + + This allows to do a lot of things just like that: + Multi-project builds, source dependencies, builds of builds and + allowing tests simply as dependent projects of the main project, etc. + + 2 Maven dependencies + I wrote my own 50 LOC Maven resolver. It's super quick and I have + yet see it not to being able to handle something. I know cases + exist, but seem rare. + alexarchambault's Coursier can be used as a more complete drop-in. + +Build scripts also have access to a small unsurprising library for +- triggering dependencies to build / download and get the classpath +- compiling Java / Scala code using zinc with given class paths +- running code +- packaging jars +- signing / publishing to sonatype/maven + +Missing features in comparison with SBT + +Not implemented yet, but rather easily possible without API changes or +major refactors is concurrently building / downloading dependencies and +running tests. Right now it is sequential. + +SBT in comparison goes a step further and is conceptually able to also run tasks +within single project to run concurrently. If I wanted to do that in +CBT, it would require an API change. I would need to make tasks Monads, +which means wrapper types and comprehensions everywhere. That would +complicate ease of use and I am actually skeptical about the benefit. +How often to you need to run tasks in parallel within a single project? +"compile" happens before "package" before "publish". + +CBT allows tasks to lazily depend on other tasks. +SBT currently does not, because it uses an Applicative, not a Monad, so +task dependencies are eager. + +Another edge case that may need a solution is dynamically overwriting +tasks. SBT allows that. Classes and traits are static. The only use +cases I know are debugging, cross builds and the sbt release plugin. A +solution could be code generating traits at build-time and mixing them +in ad-hoc. It's a build-tool after all. Build-time code-generation and +class loading is not rocket science. But there may be simpler solutions +for the cases at hand. And they are edge cases anyways. diff --git a/TODO.txt b/TODO.txt new file mode 100644 index 0000000..7f29fd1 --- /dev/null +++ b/TODO.txt @@ -0,0 +1,52 @@ +TODO: + - in progress + - automated tests + - improve logging + + - bugs + - condition guarding zinc is too eager, needs to invalid + + - immediate features + - add another class that makes all pom required fields abstract + - fix main project main method being run during tests + - DI lib into depencies + - fix conflicts in classpath stemming from dependencies + - cleanup classpath/classloader stuff + - investigate and solve multiple compilations of the same SourceDependency Build. Maybe introduce global Build map. + + - cleanup + - defs for all tasks and cached where needed + - unify work classpath + - unify argument order + + - near future features + - strip out ammonite dependency + - make cbt's own re-build concurrency safe + - unify with sbts key names where sensible + - allow updating snapshots + - cbt cli options inject add dependencies into default build + - write cached macro + - add "debug" mode that shows lots of logging + - running subproject tasks in parallel + http://stackoverflow.com/questions/743288/java-synchronization-utility + - dependency exclusion, etc. + - cache class loader per dependency in global, synchronized mutable Map + - use cli friendly responses by default everywhere + - class path debugging + - duplicate class detection + - missing/broken jars detection + - invalid files in lib folder + - integrate / build out maven search + - proper exit codes + - use zinc nailgun multi platform nailgun wrapper https://github.com/typesafehub/zinc/tree/7af98ba11d27d7667301c2222c1e702c7092bc44/src/universal/bin + + + - future features + - loop compiling with cancelling running runs/compiles + - shell tab completion + - maybe scripts for bash/zsh/fish + - maybe interactive shell + - maybe one that exists immediately after execution + + - potential features + - running in-project tasks in parallel using Monad diff --git a/bootstrap_scala/BootstrapScala.java b/bootstrap_scala/BootstrapScala.java new file mode 100644 index 0000000..fb6bd1e --- /dev/null +++ b/bootstrap_scala/BootstrapScala.java @@ -0,0 +1,105 @@ +import java.io.*; +import java.lang.reflect.*; +import java.math.*; +import java.net.*; +import java.nio.*; +import java.nio.file.*; +import java.security.*; +import java.util.*; +import java.util.stream.*; +import javax.tools.*; +import javax.xml.bind.annotation.adapters.HexBinaryAdapter; + +/* +This file class allows bootstrapping out of Java into Scala. +It downloads the Scala jars for the version number given as the first +argument into the directory given as the second argument and returns +a classpath String. +*/ +public class BootstrapScala{ + public final static Dependency[] dependencies(String target, String scalaVersion){ + return new Dependency[]{ + Dependency.scala(target, scalaVersion, "library", "f75e7acabd57b213d6f61483240286c07213ec0e"), + Dependency.scala(target, scalaVersion, "compiler","1454c21d39a4d991006a2a47c164f675ea1dafaf"), + Dependency.scala(target, scalaVersion, "reflect", "bf1649c9d33da945dea502180855b56caf06288c"), + new Dependency(target, "modules/scala-xml_2.11/1.0.5", "scala-xml_2.11-1.0.5", "77ac9be4033768cf03cc04fbd1fc5e5711de2459") + }; + } + + public static void main(String args[]){ + if(args.length < 2){ + System.err.println("Usage: bootstrap_scala <scala version> <download directory>"); + System.exit(1); + } + Dependency[] ds = dependencies( args[1], args[0] ); + + try { + new File(args[1]).mkdirs(); + Arrays.stream(ds).forEach( d -> { + download( d.url, d.path, d.hash ); + }); + System.out.println( + String.join( + File.pathSeparator, + Arrays.stream(ds).map( d -> d.path.toString() ).toArray(String[]::new) + ) + ); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + public static void download(URL urlString, Path target, String sha1){ + try { + final Path unverified = Paths.get(target+".unverified"); + + if(!Files.exists(target)){ + new File(target.toString()).getParentFile().mkdirs(); + System.err.println("downloading "+urlString); + System.err.println("to "+target); + InputStream stream = urlString.openStream(); + Files.copy(stream, unverified, StandardCopyOption.REPLACE_EXISTING); + stream.close(); + String checksum = sha1(Files.readAllBytes(unverified)); + if(sha1 == null || sha1.toUpperCase().equals(checksum)){ + Files.move(unverified, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } else { + System.err.println(target + " checksum does not match.\nExpected: |"+sha1+"|\nFound: |"+checksum+"|"); + System.exit(1); + } + } + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + public static String sha1(byte[] bytes){ + try { + final MessageDigest sha1 = MessageDigest.getInstance("SHA1"); + sha1.update( bytes, 0, bytes.length ); + return (new HexBinaryAdapter()).marshal(sha1.digest()); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } +} + + +class Dependency{ + URL url; + Path path; + String hash; + public Dependency(String target, String folder, String file, String hash){ + path = Paths.get(target+file+".jar"); + try { url = new URL("https://repo1.maven.org/maven2/org/scala-lang/"+folder+"/"+file+".jar"); } + catch (final MalformedURLException e) { throw new RuntimeException(e); } + hash = hash; + } + // scala-lang dependency + public static Dependency scala(String target, String scalaVersion, String scalaModule, String hash){ + return new Dependency( + target, + "scala-"+scalaModule+"/"+scalaVersion, + "scala-"+scalaModule+"-"+scalaVersion, + hash + ); + } +} diff --git a/bootstrap_scala/bootstrap_scala b/bootstrap_scala/bootstrap_scala new file mode 100755 index 0000000..051f429 --- /dev/null +++ b/bootstrap_scala/bootstrap_scala @@ -0,0 +1,27 @@ +#!/bin/sh + +_DIR=$(dirname $(readlink "$0") 2>/dev/null || dirname "$0" 2>/dev/null ) +DIR=$(dirname $($_DIR/../realpath/realpath.sh $0)) +JAVAC="javac -Xlint:deprecation" +TARGET=$DIR/target +CLASSES=$TARGET/classes/ +VERSION=$1 +CACHE=$DIR/cache/$VERSION/ + +COMPILER_JAR=scala-compiler-$VERSION.jar +LIBRARY_JAR=scala-library-$VERSION.jar +REFLECT_JAR=scala-reflect-$VERSION.jar +XML_JAR=scala-xml_2.11-1.0.5.jar # this is a bit fishy, because it doesn't take version into account + +mkdir -p $CLASSES + +if [ ! -f $CACHE$COMPILER_JAR ] || [ ! -f $CACHE$LIBRARY_JAR ] || [ ! -f $CACHE$REFLECT_JAR ]\ + || [ ! -f $CACHE$XML_JAR ] || [ $DIR/BootstrapScala.java -nt $CLASSES/BootstrapScala.class ] +then + echo "Recompiling CBT BootstrapScala.java" 1>&2 + $JAVAC -d $CLASSES $DIR/BootstrapScala.java + java -cp $CLASSES BootstrapScala $1 $CACHE +else + # for speedup + echo `for f in $CACHE*; do printf "$f "; done`|tr " " ":" +fi @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +#gdate +"%T.%N" +# Launcher bash script that bootstraps CBT from source. +# (Some of the code for reporting missing dependencies and waiting for nailgun to come up is a bit weird.) +# This is inentionally kept as small as posible. +# Welcome improvements to this file: +# - reduce code size through better ideas +# - reduce code size by moving more of this into type-checked Java/Scala code (if possible without performance loss). +# - reduction of dependencies +# - performance improvements + +which javac 2>&1 > /dev/null +javac_installed=$? +if [ ! $javac_installed -eq 0 ]; then + echo "You need to install javac! CBT needs it to bootstrap from Java sources into Scala." 2>&1 + exit 1 +fi +which ng 2>&1 > /dev/null +ng_installed=$? +which ng-server 2>&1 > /dev/null +ng_server_installed=$? +nailgun_installed=0 +if [ ! $ng_installed -eq 0 ] || [ ! $ng_server_installed -eq 0 ]; then + nailgun_installed=1 + echo "(Note: nailgun not found. It makes CBT faster! Try 'brew install nailgun'.)" 2>&1 +fi +which realpath 2>&1 > /dev/null +realpath_installed=$? +which gcc 2>&1 > /dev/null +gcc_installed=$? +if [ ! $realpath_installed -eq 0 ] && [ ! $gcc_installed -eq 0 ]; then + echo "You need realpath or gcc installed! CBT needs it to locate itself reliably." 2>&1 + exit 1 +fi + +which gpg 2>&1 > /dev/null +gpg_installed=$? +if [ ! $gpg_installed -eq 0 ]; then + echo "(Note: gpg not found. In order to use publishSigned you'll need it.)" 2>&1 +fi + +NAILGUN_PORT=4444 +NG="ng --nailgun-port $NAILGUN_PORT" + +CWD=$(pwd) +_DIR=$(dirname $(readlink "$0") 2>/dev/null || dirname "$0" 2>/dev/null ) +# find out real path. Build realpath if needed. +export CBT_HOME=$(dirname $($_DIR/realpath/realpath.sh $0)) + +#gdate +"%T.%N" +export SCALA_VERSION="2.11.7" +export NAILGUN=$CBT_HOME/nailgun_launcher/ +export STAGE1=$CBT_HOME/stage1/ +export TARGET=target/scala-2.11/classes/ +INDICATOR=$STAGE1$TARGET/cbt/Stage1.class + +mkdir -p $NAILGUN$TARGET +mkdir -p $STAGE1$TARGET + +which nc 2>&1 > /dev/null +nc_installed=$? + +server_up=1 +if [ $nc_installed -eq 0 ]; then + nc -z -n -w 1 127.0.0.1 $NAILGUN_PORT > /dev/null 2>&1 + server_up=$? +else + echo "(Note: nc not found. It will make slightly startup faster.)" 2>&1 +fi + +if [ ! $nc_installed -eq 0 ] || [ ! $server_up -eq 0 ]; then + echo "Starting up nailgun" 2>&1 + # try to start nailgun-server, just in case it's not up + ng-server 127.0.0.1:$NAILGUN_PORT >> $NAILGUN/target/nailgun.stdout.log 2>> $NAILGUN/target/nailgun.stderr.log & +fi + +#gdate +"%T.%N" +# fetch / find scala jars +export SCALA_CLASSPATH=`$CBT_HOME/bootstrap_scala/bootstrap_scala $SCALA_VERSION` +if [ ! $? -eq 0 ]; then echo "Problem with bootstrap_scala" 2>&1; exit 1; fi + +#gdate +"%T.%N" +# detect source changes in CBT itself +changed=0 +for file in `ls $NAILGUN/* $STAGE1/*` +do + if [ $file -nt $INDICATOR ] + then changed=1 + fi +done + +compiles1=0 +compiles2=0 + +# recompile CBT itself if needed +if [ ! $changed -eq 0 ] +then + echo "Recompiling CBT. Detected source changes..." 1>&2 + javac -Xlint:deprecation -d $NAILGUN$TARGET `ls $NAILGUN/*.java` + compiles1=$? + + rm $STAGE1$TARGET/cbt/*.class 2>/dev/null + + java -Xmx256M -Xms32M\ + -Xbootclasspath/a:$SCALA_CLASSPATH\ + -Dscala.usejavacp=true\ + -Denv.emacs=\ + scala.tools.nsc.Main\ + -deprecation\ + -feature\ + -cp $NAILGUN$TARGET\ + -d $STAGE1$TARGET\ + `ls $STAGE1/*.scala` + compiles2=$? + echo "Stopping nailgun" 2>&1 + $NG ng-stop >> $NAILGUN/target/nailgun.stdout.log 2>> $NAILGUN/target/nailgun.stderr.log & + echo "Restarting nailgun" 2>&1 + ng-server 127.0.0.1:$NAILGUN_PORT >> $NAILGUN/target/nailgun.stdout.log 2>> $NAILGUN/target/nailgun.stderr.log & +fi + +build () +{ + CP=$STAGE1$TARGET:$SCALA_CLASSPATH + if [ $nailgun_installed -eq 1 ] || [ "$1" = "publishSigned" ] || [ "$2" = "publishSigned" ] || [ "$1" = "direct" ] || [ "$2" = "direct" ] + then + #echo "Running jvm directly" 1>&2 + # -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:5005 + java -cp $NAILGUN$TARGET cbt.NailgunLauncher $mainClass $CP "$CWD" $* + else + #echo "Running via nailgun" 1>&2 + #gdate +"%T.%N" + $NG ng-cp $NAILGUN$TARGET >> $NAILGUN/target/nailgun.stdout.log 2>> $NAILGUN/target/nailgun.stderr.log + #gdate +"%T.%N" + $NG cbt.NailgunLauncher cbt.CheckAlive $CP "$CWD" $* >> $NAILGUN/target/nailgun.stdout.log 2>> $NAILGUN/target/nailgun.stderr.log + #gdate +"%T.%N" + alive=$? + while [[ $alive -ne 33 ]] + do + echo "Waiting for nailgun to start..." 1>&2 + sleep 1 + $NG ng-cp $NAILGUN$TARGET >> $NAILGUN/target/nailgun.stdout.log 2>> $NAILGUN/target/nailgun.stderr.log + $NG cbt.NailgunLauncher cbt.CheckAlive $CP "$CWD" $* >> $NAILGUN/target/nailgun.stdout.log 2>> $NAILGUN/target/nailgun.stderr.log + alive=$? + done + #gdate +"%T.%N" + $NG cbt.NailgunLauncher $mainClass $CP "$CWD" $* + #gdate +"%T.%N" + + fi +} +#gdate +"%T.%N" +# run CBT and loop if desired. This allows recompiling CBT itself as part of compile looping. +if [ $compiles1 -eq 0 ] && [ $compiles2 -eq 0 ] +then + if [ "$1" = "admin" ]; then + mainClass=cbt.AdminStage1 + else + mainClass=cbt.Stage1 + fi + build $* + if [ "$1" = "loop" ] + then + while true; do + echo "======= Restarting CBT =======" 2>&1 + build $* + done + fi +fi +#gdate +"%T.%N"
\ No newline at end of file diff --git a/coursier/Coursier.scala b/coursier/Coursier.scala new file mode 100644 index 0000000..a2f5f39 --- /dev/null +++ b/coursier/Coursier.scala @@ -0,0 +1,47 @@ +/* +package cbt +object Coursier{ + implicit class CoursierDependencyResolution(d: MavenDependency){ + import d._ + def resolveCoursier = { + import coursier._ + val repositories = Seq( + Cache.ivy2Local, + MavenRepository("https://repo1.maven.org/maven2") + ) + + val start = Resolution( + Set( + MavenDependency( + Module(groupId, artifactId), version + ) + ) + ) + + val fetch = Fetch.from(repositories, Cache.fetch()) + + + val resolution = start.process.run(fetch).run + + val errors: Seq[(MavenDependency, Seq[String])] = resolution.errors + + if(errors.nonEmpty) throw new Exception(errors.toString) + + import java.io.File + import scalaz.\/ + import scalaz.concurrent.Task + + val localArtifacts: Seq[FileError \/ File] = Task.gatherUnordered( + resolution.artifacts.map(Cache.file(_).run) + ).run + + val files = localArtifacts.map(_.toEither match { + case Left(error) => throw new Exception(error.toString) + case Right(file) => file + }) + + resolution.dependencies.map( d => cbt.MavenDependency(d.module.organization,d.module.name, d.version)).to[collection.immutable.Seq] + } + } +} +*/
\ No newline at end of file diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java new file mode 100644 index 0000000..a6858d9 --- /dev/null +++ b/nailgun_launcher/NailgunLauncher.java @@ -0,0 +1,48 @@ +package cbt; +import java.io.*; +import java.lang.reflect.*; +import java.net.*; +import java.nio.*; +import java.nio.file.*; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/* +This launcher allows to use Nailgun without loading anything else permanenetly into it's classpath. +The main method loads the given class from the given class math, calls it's main methods passing +in the additional arguments. +*/ +public class NailgunLauncher{ + /* Persistent cache for caching classloaders for the JVM life time. + Can be used as needed by user code to improve startup time. */ + public static ConcurrentHashMap<String,ClassLoader> classLoaderCache = new ConcurrentHashMap<String,ClassLoader>(); + public static void main(String[] args){ + try{ + if(args.length < 3){ + System.out.println("usage: <main class> <class path> <... args>"); + } else { + URLClassLoader cl = new URLClassLoader( // TODO: cache this classloader, but invalidate on changes + Arrays.stream( + args[1].split(File.pathSeparator) + ).filter( cp -> !(cp == "") ).map( cp -> { + try { return new URL("file:" + cp); } + catch(MalformedURLException e) { throw new RuntimeException(e); } + }).toArray(URL[]::new) + ){ + public String toString(){ + String suffix = ""; + if(getParent() != ClassLoader.getSystemClassLoader()) + suffix = ", "+getParent(); + return "URLClassLoader(" + Arrays.toString(getURLs()) + suffix +")"; + } + }; + cl .loadClass( args[0] ) + .getMethod( "main", String[].class ) + .invoke( + null/* _cls.newInstance()*/, + (Object) Arrays.stream(args).skip(2).toArray( String[]::new ) + ); + } + } catch (Exception e) { throw new RuntimeException(e); } + } +} diff --git a/realpath/realpath.c b/realpath/realpath.c new file mode 100644 index 0000000..055dbcf --- /dev/null +++ b/realpath/realpath.c @@ -0,0 +1,35 @@ +// http://stackoverflow.com/questions/284662/how-do-you-normalize-a-file-path-in-bash +// realpath.c: display the absolute path to a file or directory. +// Adam Liss, August, 2007 +// This program is provided "as-is" to the public domain, without express or +// implied warranty, for any non-profit use, provided this notice is maintained. + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <libgen.h> +#include <limits.h> + +static char *s_pMyName; +void usage(void); + +int main(int argc, char *argv[]) +{ + char + sPath[PATH_MAX]; + + + s_pMyName = strdup(basename(argv[0])); + + if (argc < 2) + usage(); + + printf("%s\n", realpath(argv[1], sPath)); + return 0; +} + +void usage(void) +{ + fprintf(stderr, "usage: %s PATH\n", s_pMyName); + exit(1); +}
\ No newline at end of file diff --git a/realpath/realpath.sh b/realpath/realpath.sh new file mode 100755 index 0000000..de4d964 --- /dev/null +++ b/realpath/realpath.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# is there a realiable cross-platform was to do this without relying on compiling C code? + +DIR=$(dirname $(readlink "$0") 2>/dev/null || dirname "$0" 2>/dev/null ) +which realpath 2>&1 > /dev/null +REALPATH_INSTALLED=$? + +if [ ! $REALPATH_INSTALLED -eq 0 ]; then + if [ ! -f $DIR/realpath ]; then + >&2 echo "Compiling realpath" + gcc $DIR/realpath.c -o $DIR/realpath + chmod u+x $DIR/realpath + fi + $DIR/realpath $1 +else + realpath $1 +fi diff --git a/stage1/Cache.scala b/stage1/Cache.scala new file mode 100644 index 0000000..6e6b9eb --- /dev/null +++ b/stage1/Cache.scala @@ -0,0 +1,14 @@ +package cbt +/** +Caches exactly one value +Is there a less boiler-platy way to achieve this, that doesn't +require creating an instance for each thing you want to cache? +*/ +class Cache[T]{ + private var value: Option[T] = None + def apply(value: => T) = this.synchronized{ + if(!this.value.isDefined) + this.value = Some(value) + this.value.get + } +} diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala new file mode 100644 index 0000000..13c097e --- /dev/null +++ b/stage1/Stage1.scala @@ -0,0 +1,70 @@ +package cbt +import java.io._ +import paths._ +import scala.collection.immutable.Seq + +object CheckAlive{ + def main(args: Array[String]): Unit = { + System.exit(33) + } +} +object Stage1 extends Stage1Base{ + def mainClass = ("cbt.Stage2") +} +object AdminStage1 extends Stage1Base{ + def mainClass = ("cbt.AdminStage2") +} +abstract class Stage1Base{ + class Init(args: Array[String]){ + import scala.collection.JavaConverters._ + val propsRaw: Seq[String] = args.toVector.filter(_.startsWith("-D")) + val argsV: Seq[String] = args.toVector diff propsRaw + + lazy val props = propsRaw.map(_.drop(2)).map(_.split("=")).map{ + case Array(key, value) => key -> value + }.toMap ++ System.getProperties.asScala + val logger = new Logger(props.get("log")) + + val cwd = argsV(0) + } + def mainClass: String + def main(args: Array[String]): Unit = { + import java.time.LocalTime.now + val init = new Init(args) + val lib = new Stage1Lib(init.logger) + lib.logger.stage1(s"[$now] Stage1 start") + lib.logger.stage1("Stage1: after creating lib") + import lib._ + val cwd = args(0) + + val src = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) + val changeIndicator = new File(stage2Target+"/cbt/Build.class") + + def newerThan( a: File, b: File ) ={ + val res = a.lastModified > b.lastModified + if(res) { + /* + println(a) + println(a.lastModified) + println(b) + println(b.lastModified) + */ + } + res + } + + logger.stage1("before conditionally running zinc to recompile CBT") + if( src.exists(newerThan(_, changeIndicator)) ){ + val stage1Classpath = CbtDependency(init.logger).dependencyClasspath + logger.stage1("cbt.lib has changed. Recompiling with cp: "+stage1Classpath) + lib.zinc( true, src, stage2Target, stage1Classpath )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion ) + } + logger.stage1(s"[$now] calling CbtDependency.classLoader") + + logger.stage1(s"[$now] Run Stage2") + lib.runMain( mainClass, cwd +: args.drop(1).toVector, CbtDependency(init.logger).classLoader ) + lib.logger.stage1(s"[$now] Stage1 end") + + + } +} diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala new file mode 100644 index 0000000..cc8a5e3 --- /dev/null +++ b/stage1/Stage1Lib.scala @@ -0,0 +1,184 @@ +package cbt + +import cbt.paths._ + +import java.io._ +import java.net._ +import java.nio.file._ +import javax.tools._ +import java.security._ +import java.util._ +import javax.xml.bind.annotation.adapters.HexBinaryAdapter + +import scala.collection.immutable.Seq + +case class Context( cwd: String, args: Seq[String], logger: Logger ) + +case class ClassPath(files: Seq[File]){ + private val duplicates = (files diff files.distinct).distinct + assert( + duplicates.isEmpty, + "Duplicate classpath entries found:\n" + duplicates.mkString("\n") + "\nin classpath:\n"+string + ) + private val nonExisting = files.distinct.filterNot(_.exists) + assert( + duplicates.isEmpty, + "Classpath contains entires that don't exist on disk:\n" + nonExisting.mkString("\n") + "\nin classpath:\n"+string + ) + + def +:(file: File) = ClassPath(file +: files) + def :+(file: File) = ClassPath(files :+ file) + def ++(other: ClassPath) = ClassPath(files ++ other.files) + def string = strings.mkString( File.pathSeparator ) + def strings = files.map{ + f => f.toString + ( if(f.isDirectory) "/" else "" ) + } + def toConsole = string +} +object ClassPath{ + def flatten( classPaths: Seq[ClassPath] ): ClassPath = ClassPath( classPaths.map(_.files).flatten ) +} + +class Stage1Lib( val logger: Logger ){ + lib => + + // ========== reflection ========== + + /** Create instance of the given class via reflection */ + def create(cls: String)(args: Any*)(classLoader: ClassLoader): Any = { + logger.composition( logger.showInvocation("Stage1Lib.create", (classLoader,cls,args)) ) + import scala.reflect.runtime.universe._ + val m = runtimeMirror(classLoader) + val sym = m.classSymbol(classLoader.loadClass(cls)) + val cm = m.reflectClass( sym.asClass ) + val tpe = sym.toType + val ctorm = cm.reflectConstructor( tpe.decl(termNames.CONSTRUCTOR).asMethod ) + ctorm(args:_*) + } + + // ========== file system / net ========== + + def array2hex(padTo: Int, array: Array[Byte]): String = { + val hex = new java.math.BigInteger(1, array).toString(16) + ("0" * (padTo-hex.size)) + hex + } + def md5( bytes: Array[Byte] ): String = array2hex(32, MessageDigest.getInstance("MD5").digest(bytes)) + def sha1( bytes: Array[Byte] ): String = array2hex(40, MessageDigest.getInstance("SHA-1").digest(bytes)) + + def red(string: String) = scala.Console.RED+string+scala.Console.RESET + def blue(string: String) = scala.Console.BLUE+string+scala.Console.RESET + def green(string: String) = scala.Console.GREEN+string+scala.Console.RESET + + def download(urlString: URL, target: Path, sha1: Option[String]){ + val incomplete = Paths.get(target+".incomplete"); + if( !Files.exists(target) ){ + new File(target.toString).getParentFile.mkdirs + logger.resolver(blue("downloading ")+urlString) + logger.resolver(blue("to ")+target) + val stream = urlString.openStream + Files.copy(stream, incomplete, StandardCopyOption.REPLACE_EXISTING) + sha1.foreach{ + hash => + val expected = hash + val actual = this.sha1(Files.readAllBytes(incomplete)) + assert( expected == actual, s"$expected == $actual" ) + logger.resolver(green("verified")+" checksum for "+target) + } + stream.close + Files.move(incomplete, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } + } + + def listFilesRecursive(f: File): Seq[File] = { + f +: ( + if( f.isDirectory ) f.listFiles.flatMap(listFilesRecursive).toVector else Seq[File]() + ) + } + + // ========== compilation / execution ========== + + def runMainIfFound(cls: String, args: Seq[String], classLoader: ClassLoader ){ + if( classLoader.canLoad(cls) ) runMain(cls: String, args: Seq[String], classLoader: ClassLoader ) + } + + def runMain(cls: String, args: Seq[String], classLoader: ClassLoader ){ + logger.lib(s"Running $cls.main($args) with classLoader: "+classLoader) + classLoader + .loadClass(cls) + .getMethod( "main", scala.reflect.classTag[Array[String]].runtimeClass ) + .invoke( null, args.toArray.asInstanceOf[AnyRef] ); + } + + implicit class ClassLoaderExtensions(classLoader: ClassLoader){ + def canLoad(className: String) = { + try{ + classLoader.loadClass(className) + true + } catch { + case e: ClassNotFoundException => false + } + } + } + + def zinc( + needsRecompile: Boolean, files: Seq[File], compileTarget: File, classpath: ClassPath, extraArgs: Seq[String] = Seq() + )( zincVersion: String, scalaVersion: String ): Unit = { + + val cp = classpath.string + if(classpath.files.isEmpty) throw new Exception("Trying to compile with empty classpath. Source files: "+files) + if(files.isEmpty) throw new Exception("Trying to compile no files. ClassPath: "+cp) + + // only run zinc if files changed, for performance reasons + // FIXME: this is broken, need invalidate on changes in dependencies as well + if( true || needsRecompile ){ + val zinc = MavenDependency("com.typesafe.zinc","zinc", zincVersion)(logger) + val zincDeps = zinc.transitiveDependencies + + val sbtInterface = + zincDeps + .collect{ case d @ MavenDependency( "com.typesafe.sbt", "sbt-interface", _, false ) => d } + .headOption + .getOrElse( throw new Exception(s"cannot find sbt-interface in zinc $zincVersion dependencies") ) + .jar + + val compilerInterface = + zincDeps + .collect{ case d @ MavenDependency( "com.typesafe.sbt", "compiler-interface", _, true ) => d } + .headOption + .getOrElse( throw new Exception(s"cannot find compiler-interface in zinc $zincVersion dependencies") ) + .jar + + val scalaLibrary = MavenDependency("org.scala-lang","scala-library",scalaVersion)(logger).jar + val scalaReflect = MavenDependency("org.scala-lang","scala-reflect",scalaVersion)(logger).jar + val scalaCompiler = MavenDependency("org.scala-lang","scala-compiler",scalaVersion)(logger).jar + + redirectOutToErr{ + lib.runMain( + "com.typesafe.zinc.Main", + Seq( + "-scala-compiler", scalaCompiler.toString, + "-scala-library", scalaLibrary.toString, + "-sbt-interface", sbtInterface.toString, + "-compiler-interface", compilerInterface.toString, + "-scala-extra", scalaReflect.toString, + "-cp", cp, + "-d", compileTarget.toString + ) ++ extraArgs.map("-S"+_) ++ files.map(_.toString), + zinc.classLoader + ) + } + } + + } + def redirectOutToErr[T](code: => T): Unit = { + val oldOut = System.out + try{ + System.setOut(System.err) + code + } finally{ + System.setOut(oldOut) + } + } + +} + diff --git a/stage1/classloader.scala b/stage1/classloader.scala new file mode 100644 index 0000000..6f2213b --- /dev/null +++ b/stage1/classloader.scala @@ -0,0 +1,65 @@ +package cbt +import java.io._ +import java.net._ +import java.nio.file._ +import scala.util.Try + +import scala.collection.immutable.Seq + +object ClassLoaderCache{ + private val cache = NailgunLauncher.classLoaderCache + def classLoader( path: String, parent: ClassLoader ): ClassLoader = { + def realpath( name: String ) = Paths.get(new File(name).getAbsolutePath).normalize.toString + val normalized = realpath(path) + if( cache.containsKey(normalized) ){ + //println("FOUND: "+normalized) + cache.get(normalized) + } else { + //println("PUTTING: "+normalized) + //Try(???).recover{ case e=>e.printStackTrace} + val cl = new cbt.URLClassLoader( ClassPath(Seq(new File(normalized))), parent ) + cache.put( normalized, cl ) + cl + } + } + def remove( path: String ) = cache.remove( path ) +} +class MultiClassLoader(parents: Seq[ClassLoader]) extends ClassLoader { + override def loadClass(name: String) = { + //System.err.println("LOADING CLASS "+name); + val c = parents.toStream.map{ + parent => + Try{ + parent.loadClass(name) + }.map(Option[Class[_]](_)).recover{ + case _:ClassNotFoundException => None + }.get + }.find(_.isDefined).flatten + c.getOrElse( ClassLoader.getSystemClassLoader.loadClass(name) ) + } + override def toString = "MultiClassLoader(" + parents.mkString(",") + ")" +} +case class URLClassLoader(classPath: ClassPath, parent: ClassLoader) + extends java.net.URLClassLoader( + classPath.strings.map( + path => new URL("file:"+path) + ).toArray, + parent + ){ + override def toString = ( + scala.Console.BLUE + "cbt.URLClassLoader" + scala.Console.RESET + "(\n " + getURLs.map(_.toString).sorted.mkString(",\n ") + + (if(getParent() != ClassLoader.getSystemClassLoader()) ",\n" + getParent().toString.split("\n").map(" "+_).mkString("\n") else "") + + "\n)" + ) + import scala.language.existentials + /*override def loadClass(name: String): Class[_] = { + //System.err.println("LOADING CLASS "+name+" in "+this); + try{ + super.loadClass(name) + } catch { + case e: ClassNotFoundException => + // FIXME: Shouldn't this happen automatically? + parent.loadClass(name) + } + }*/ +} diff --git a/stage1/constants.scala b/stage1/constants.scala new file mode 100644 index 0000000..bf0943e --- /dev/null +++ b/stage1/constants.scala @@ -0,0 +1,4 @@ +package cbt +object constants{ + def scalaVersion = Option(System.getenv("SCALA_VERSION")).get +} diff --git a/stage1/logger.scala b/stage1/logger.scala new file mode 100644 index 0000000..16bd940 --- /dev/null +++ b/stage1/logger.scala @@ -0,0 +1,41 @@ +package cbt +import java.time._ +// We can replace this with something more sophisticated eventually +case class Logger(enabledLoggers: Set[String]){ + val start = LocalTime.now() + //System.err.println("Created Logger("+enabledLoggers+")") + def this(enabledLoggers: Option[String]) = this( enabledLoggers.toVector.flatMap( _.split(",") ).toSet ) + def log(name: String, msg: => String) = { + val timeTaken = (Duration.between(start, LocalTime.now()).toMillis.toDouble / 1000).toString + System.err.println( s"[${" "*(6-timeTaken.size)}$timeTaken]["+name+"] " + msg ) + } + + def showInvocation(method: String, args: Any) = method + "( " + args + " )" + + final def stage1(msg: => String) = logGuarded(names.stage1, msg) + final def stage2(msg: => String) = logGuarded(names.stage2, msg) + final def loop(msg: => String) = logGuarded(names.loop, msg) + final def task(msg: => String) = logGuarded(names.task, msg) + final def composition(msg: => String) = logGuarded(names.composition, msg) + final def resolver(msg: => String) = logGuarded(names.resolver, msg) + final def lib(msg: => String) = logGuarded(names.lib, msg) + + private object names{ + val stage1 = "stage1" + val stage2 = "stage2" + val loop = "loop" + val task = "task" + val resolver = "resolver" + val composition = "composition" + val lib = "lib" + } + + private def logGuarded(name: String, msg: => String) = { + if( + (enabledLoggers contains name) + || (enabledLoggers contains "all") + ){ + log(name, msg) + } + } +}
\ No newline at end of file diff --git a/stage1/paths.scala b/stage1/paths.scala new file mode 100644 index 0000000..f76c2f7 --- /dev/null +++ b/stage1/paths.scala @@ -0,0 +1,15 @@ +package cbt +import java.io._ +object paths{ + val cbtHome = new File(Option(System.getenv("CBT_HOME")).get) + val mavenCache = new File(cbtHome+"/cache/maven/") + val userHome = new File(Option(System.getProperty("user.home")).get) + val stage1 = new File(Option(System.getenv("STAGE1")).get) + val stage2 = new File(cbtHome + "/stage2/") + val nailgun = new File(Option(System.getenv("NAILGUN")).get) + private val target = Option(System.getenv("TARGET")).get + val stage1Target = new File(stage1 + "/" + target) + val stage2Target = new File(stage2 + "/" + target) + val nailgunTarget = new File(nailgun + "/" + target) + val sonatypeLogin = new File(cbtHome+"/sonatype.login") +} diff --git a/stage1/resolver.scala b/stage1/resolver.scala new file mode 100644 index 0000000..880289c --- /dev/null +++ b/stage1/resolver.scala @@ -0,0 +1,264 @@ +package cbt +import java.nio.file._ +import java.net._ +import java.io._ +import scala.collection.immutable.Seq +import scala.xml._ +import paths._ + +private final class Tree( val root: Dependency, computeChildren: => Seq[Tree] ){ + lazy val children = computeChildren + def linearize: Seq[Dependency] = root +: children.flatMap(_.linearize) + def show(indent: Int = 0): Stream[Char] = { + (" " * indent + root.show + "\n").toStream #::: children.map(_.show(indent+1)).foldLeft(Stream.empty[Char])(_ #::: _) + } +} + +trait ArtifactInfo extends Dependency{ + def artifactId: String + def groupId: String + def version: String + + protected def str = s"$groupId:$artifactId:$version" + override def show = super.show + s"($str)" +} +abstract class Dependency{ + + def updated: Boolean + //def cacheClassLoader: Boolean = false + def exportedClasspath: ClassPath + def exportedJars: Seq[File] + def jars: Seq[File] = exportedJars ++ dependencyJars + + def cacheDependencyClassLoader = true + + private object cacheClassLoaderBasicBuild extends Cache[URLClassLoader] + def classLoader: URLClassLoader = cacheClassLoaderBasicBuild{ + val transitiveClassPath = transitiveDependencies.map{ + case d: MavenDependency => Left(d) + case d => Right(d) + } + val buildClassPath = ClassPath.flatten( + transitiveClassPath.flatMap( + _.right.toOption.map(_.exportedClasspath) + ) + ) + val mavenClassPath = ClassPath.flatten( + transitiveClassPath.flatMap( + _.left.toOption + ).par.map(_.exportedClasspath).seq.sortBy(_.string) + ) + if(cacheDependencyClassLoader){ + val mavenClassPathKey = mavenClassPath.strings.sorted.mkString(":") + new URLClassLoader( + exportedClasspath ++ buildClassPath, + ClassLoaderCache.classLoader( + mavenClassPathKey, new URLClassLoader( mavenClassPath, ClassLoader.getSystemClassLoader ) + ) + ) + } else { + new URLClassLoader( + exportedClasspath ++ buildClassPath ++ mavenClassPath, ClassLoader.getSystemClassLoader + ) + } + } + def classpath : ClassPath = exportedClasspath ++ dependencyClasspath + def dependencyJars : Seq[File] = transitiveDependencies.flatMap(_.jars) + def dependencyClasspath : ClassPath = ClassPath.flatten( transitiveDependencies.map(_.exportedClasspath) ) + def dependencies: Seq[Dependency] + + private def resolveRecursive(parents: List[Dependency] = List()): Tree = { + // diff removes circular dependencies + new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents))) + } + + def transitiveDependencies: Seq[Dependency] = { + val deps = dependencies.flatMap(_.resolveRecursive().linearize) + val hasInfo = deps.collect{ case d:ArtifactInfo => d } + val noInfo = deps.filter{ + case _:ArtifactInfo => false + case _ => true + } + noInfo ++ MavenDependency.removeOutdated( hasInfo ) + } + + def show: String = this.getClass.getSimpleName + // ========== debug ========== + def dependencyTree: String = dependencyTreeRecursion() + def logger: Logger + protected def lib = new Stage1Lib(logger) + private def dependencyTreeRecursion(indent: Int = 0): String = ( " " * indent ) + (if(updated) lib.red(show) else show) + dependencies.map(_.dependencyTreeRecursion(indent + 1)).map("\n"+_).mkString("") + + private object cacheDependencyClassLoaderBasicBuild extends Cache[ClassLoader] +} + +// TODO: all this hard codes the scala version, needs more flexibility +class ScalaCompiler(logger: Logger) extends MavenDependency("org.scala-lang","scala-compiler",constants.scalaVersion)(logger) +class ScalaLibrary(logger: Logger) extends MavenDependency("org.scala-lang","scala-library",constants.scalaVersion)(logger) +class ScalaReflect(logger: Logger) extends MavenDependency("org.scala-lang","scala-reflect",constants.scalaVersion)(logger) + +case class ScalaDependencies(logger: Logger) extends Dependency{ + def exportedClasspath = ClassPath(Seq()) + def exportedJars = Seq[File]() + def dependencies = Seq( new ScalaCompiler(logger), new ScalaLibrary(logger), new ScalaReflect(logger) ) + final val updated = false +} + +/* +case class BinaryDependency( path: File, dependencies: Seq[Dependency] ) extends Dependency{ + def exportedClasspath = ClassPath(Seq(path)) + def exportedJars = Seq[File]() +} +*/ + +case class Stage1Dependency(logger: Logger) extends Dependency{ + def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) + def exportedJars = Seq[File]() + def dependencies = ScalaDependencies(logger: Logger).dependencies + def updated = false // FIXME: think this through, might allow simplifications and/or optimizations +} +case class CbtDependency(logger: Logger) extends Dependency{ + def exportedClasspath = ClassPath( Seq( stage2Target ) ) + def exportedJars = Seq[File]() + override def dependencies = Seq( + Stage1Dependency(logger), + MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0")(logger), + MavenDependency("com.lihaoyi","ammonite-repl_2.11.7","0.5.5")(logger), + MavenDependency("org.scala-lang.modules","scala-xml_2.11","1.0.5")(logger) + ) + def updated = false // FIXME: think this through, might allow simplifications and/or optimizations +} + +sealed trait ClassifierBase +final case class Classifier(name: String) extends ClassifierBase +case object javadoc extends ClassifierBase +case object sources extends ClassifierBase + +case class MavenDependency( groupId: String, artifactId: String, version: String, sources: Boolean = false )(val logger: Logger) + extends ArtifactInfo{ + + def updated = false + + private val groupPath = groupId.split("\\.").mkString("/") + def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version"+(if(sources) "-sources" else "") + + private def resolverUrl = if(version.endsWith("-SNAPSHOT")) "https://oss.sonatype.org/content/repositories/snapshots" else "https://repo1.maven.org/maven2" + private def baseUrl = resolverUrl + basePath + private def baseFile = mavenCache + basePath + private def pomFile = baseFile+".pom" + private def jarFile = baseFile+".jar" + //private def coursierJarFile = userHome+"/.coursier/cache/v1/https/repo1.maven.org/maven2"+basePath+".jar" + private def pomUrl = baseUrl+".pom" + private def jarUrl = baseUrl+".jar" + + def exportedJars = Seq( jar ) + def exportedClasspath = ClassPath( exportedJars ) + + import scala.collection.JavaConversions._ + + def jarSha1 = { + val file = jarFile+".sha1" + def url = jarUrl+".sha1" + scala.util.Try{ + lib.download( new URL(url), Paths.get(file), None ) + // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom + Files.readAllLines(Paths.get(file)).mkString("\n").split(" ").head.trim + }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one + } + def pomSha1 = { + val file = pomFile+".sha1" + def url = pomUrl+".sha1" + scala.util.Try{ + lib.download( new URL(url), Paths.get(file), None ) + // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom + Files.readAllLines(Paths.get(file)).mkString("\n").split(" ").head.trim + }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one + } + def jar = { + lib.download( new URL(jarUrl), Paths.get(jarFile), jarSha1 ) + new File(jarFile) + } + def pomXml = { + XML.loadFile(pom.toString) + } + def pom = { + lib.download( new URL(pomUrl), Paths.get(pomFile), pomSha1 ) + new File(pomFile) + } + + // ========== pom traversal ========== + + lazy val pomParents: Seq[MavenDependency] = { + (pomXml \ "parent").collect{ + case parent => + MavenDependency( + (parent \ "groupId").text, + (parent \ "artifactId").text, + (parent \ "version").text + )(logger) + } + } + def dependencies: Seq[MavenDependency] = { + if(sources) Seq() + else (pomXml \ "dependencies" \ "dependency").collect{ + case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" => + MavenDependency( + lookup(xml,_ \ "groupId").get, + lookup(xml,_ \ "artifactId").get, + lookup(xml,_ \ "version").get, + (xml \ "classifier").text == "sources" + )(logger) + }.toVector + } + def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = { + //println("lookup in "+pomUrl) + val Substitution = "\\$\\{([a-z0-9\\.]+)\\}".r + accessor(xml).headOption.flatMap{v => + //println("found: "+v.text) + v.text match { + case Substitution(path) => + //println("lookup "+path + ": "+(pomXml\path).text) + lookup(pomXml, _ \ "properties" \ path) + case value => Option(value) + } + }.orElse( + pomParents.map(p => p.lookup(p.pomXml, accessor)).flatten.headOption + ) + } +} +object MavenDependency{ + def semanticVersionLessThan(left: String, right: String) = { + // FIXME: this ignores ends when different size + val zipped = left.split("\\.|\\-").map(toInt) zip right.split("\\.|\\-").map(toInt) + val res = zipped.map { + case (Left(i),Left(j)) => i compare j + case (Right(i),Right(j)) => i compare j + case (Left(i),Right(j)) => i.toString compare j + case (Right(i),Left(j)) => i compare j.toString + } + res.find(_ != 0).map(_ < 0).getOrElse(false) + } + def toInt(str: String): Either[Int,String] = try { + Left(str.toInt) + } catch { + case e: NumberFormatException => Right(str) + } + /* this obviously should be overridable somehow */ + def removeOutdated( + deps: Seq[ArtifactInfo], + versionLessThan: (String, String) => Boolean = semanticVersionLessThan + ): Seq[ArtifactInfo] = { + val latest = deps + .groupBy( d => (d.groupId, d.artifactId) ) + .mapValues( + _.sortBy( _.version )( Ordering.fromLessThan(versionLessThan) ) + .last + ) + deps.flatMap{ + d => + val l = latest.get((d.groupId,d.artifactId)) + //if(d != l) println("EVICTED: "+d.show) + l + }.distinct + } +} diff --git a/stage2/AdminStage2.scala b/stage2/AdminStage2.scala new file mode 100644 index 0000000..e7e2284 --- /dev/null +++ b/stage2/AdminStage2.scala @@ -0,0 +1,11 @@ +package cbt +object AdminStage2{ + def main(args: Array[String]) = { + val init = new Stage1.Init(args.drop(3)) + val lib = new Lib(init.logger) + val adminTasks = new AdminTasks(lib, args.drop(3)) + new lib.ReflectObject(adminTasks){ + def usage = "Available methods: " + lib.taskNames(subclassType) + }.callNullary(args.lift(2)) + } +} diff --git a/stage2/AdminTasks.scala b/stage2/AdminTasks.scala new file mode 100644 index 0000000..2f7efe1 --- /dev/null +++ b/stage2/AdminTasks.scala @@ -0,0 +1,12 @@ +package cbt +class AdminTasks(lib: Lib, args: Array[String]){ + def resolve = { + ClassPath.flatten( + args(0).split(",").toVector.map{ + d => + val v = d.split(":") + new MavenDependency(v(0),v(1),v(2))(lib.logger).classpath + } + ) + } +} diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala new file mode 100644 index 0000000..41589db --- /dev/null +++ b/stage2/BuildBuild.scala @@ -0,0 +1,17 @@ +package cbt +import scala.collection.immutable.Seq + +class BuildBuild(context: Context) extends Build(context){ + override def dependencies = Seq( CbtDependency(context.logger) ) ++ super.dependencies + def managedBuildDirectory = lib.realpath(projectDirectory + "/../") + val managedBuild = { + val managedContext = context.copy( cwd = managedBuildDirectory ) + val cl = new cbt.URLClassLoader( + classpath, + classOf[BuildBuild].getClassLoader // FIXME: this looks wrong. Should be ClassLoader.getSystemClassLoader but that crashes + ) + lib.create( lib.buildClassName )( managedContext )( cl ).asInstanceOf[Build] + } + override def triggerLoopFiles = super.triggerLoopFiles ++ managedBuild.triggerLoopFiles + override def finalBuild = managedBuild.finalBuild +} diff --git a/stage2/DefaultBuild.scala b/stage2/DefaultBuild.scala new file mode 100644 index 0000000..c0072ff --- /dev/null +++ b/stage2/DefaultBuild.scala @@ -0,0 +1,233 @@ +package cbt +import cbt.paths._ + +import java.io._ +import java.lang.reflect.InvocationTargetException +import java.net._ +import java.nio.file.{Path =>_,_} +import java.nio.file.Files.readAllBytes +import java.security.MessageDigest +import java.util.jar._ + +import scala.collection.immutable.Seq +import scala.reflect.runtime.{universe => ru} +import scala.util._ + +import ammonite.ops.{cwd => _,_} + + + + +abstract class PackageBuild(context: Context) extends Build(context) with ArtifactInfo{ + def `package`: Seq[File] = lib.concurrently( enableConcurrency )( + Seq(() => jar, () => docJar, () => srcJar) + )( _() ) + + private object cacheJarBasicBuild extends Cache[File] + def jar: File = cacheJarBasicBuild{ + lib.jar( artifactId, version, compile, jarTarget ) + } + + private object cacheSrcJarBasicBuild extends Cache[File] + def srcJar: File = cacheSrcJarBasicBuild{ + lib.srcJar(sources, artifactId, version, scalaTarget) + } + + private object cacheDocBasicBuild extends Cache[File] + def docJar: File = cacheDocBasicBuild{ + lib.docJar( sources, dependencyClasspath, apiTarget, jarTarget, artifactId, version, scalacOptions ) + } + + override def jars = jar +: dependencyJars + override def exportedJars: Seq[File] = Seq(jar) +} +abstract class PublishBuild(context: Context) extends PackageBuild(context){ + def name = artifactId + def description: String + def url: URL + def developers: Seq[Developer] + def licenses: Seq[License] + def scmUrl: String + def scmConnection: String + def pomExtra: Seq[scala.xml.Node] = Seq() + + // ========== package ========== + + /** put additional xml that should go into the POM file in here */ + def pom: File = lib.pom( + groupId = groupId, + artifactId = artifactId, + version = version, + name = name, + description = description, + url = url, + developers = developers, + licenses = licenses, + scmUrl = scmUrl, + scmConnection = scmConnection, + dependencies = dependencies, + pomExtra = pomExtra, + jarTarget = jarTarget + ) + + // ========== publish ========== + final protected def releaseFolder = s"/${groupId.replace(".","/")}/$artifactId/$version/" + def snapshotUrl = new URL("https://oss.sonatype.org/content/repositories/snapshots") + def releaseUrl = new URL("https://oss.sonatype.org/service/local/staging/deploy/maven2") + def publishSnapshot: Unit = lib.publishSnapshot(sourceFiles, pom +: `package`, new URL(snapshotUrl + releaseFolder) ) + def publishSigned: Unit = lib.publishSigned(sourceFiles, pom +: `package`, new URL(releaseUrl + releaseFolder) ) +} + + +class BasicBuild(context: Context) extends Build(context) +class Build(val context: Context) extends Dependency with TriggerLoop{ + // library available to builds + final val logger = context.logger + override final protected val lib: Lib = new Lib(logger) + // ========== general stuff ========== + + def enableConcurrency = false + final def projectDirectory: File = new File(context.cwd) + assert( projectDirectory.exists, "projectDirectory does not exist: "+projectDirectory ) + final def usage: Unit = new lib.ReflectBuild(this).usage +/* + def scaffold: Unit = lib.generateBasicBuildFile( + projectDirectory, scalaVersion, groupId, artifactId, version + ) +*/ + // ========== meta data ========== + + def scalaVersion: String = constants.scalaVersion + final def scalaMajorVersion: String = scalaVersion.split("\\.").take(2).mkString(".") + def zincVersion = "0.3.9" + + def dependencies: Seq[Dependency] = Seq( + "org.scala-lang" % "scala-library" % scalaVersion + ) + + // ========== paths ========== + final private val defaultSourceDirectory = new File(projectDirectory+"/src/") + + /** base directory where stuff should be generated */ + def target = new File(projectDirectory+"/target") + /** base directory where stuff should be generated for this scala version*/ + def scalaTarget = new File(target + s"/scala-$scalaMajorVersion") + /** directory where jars (and the pom file) should be put */ + def jarTarget = scalaTarget + /** directory where the scaladoc should be put */ + def apiTarget = new File(scalaTarget + "/api") + /** directory where the class files should be put (in package directories) */ + def compileTarget = new File(scalaTarget + "/classes") + + /** Source directories and files. Defaults to .scala and .java files in src/ and top-level. */ + def sources: Seq[File] = Seq(defaultSourceDirectory) ++ projectDirectory.listFiles.toVector.filter(sourceFileFilter) + + /** Which file endings to consider being source files. */ + def sourceFileFilter(file: File): Boolean = file.toString.endsWith(".scala") || file.toString.endsWith(".java") + + /** Absolute path names for all individual files found in sources directly or contained in directories. */ + final def sourceFiles: Seq[File] = for { + base <- sources.filter(_.exists).map(lib.realpath) + file <- lib.listFilesRecursive(base) if file.isFile && sourceFileFilter(file) + } yield file + + protected def assertSourceDirectories(): Unit = { + val nonExisting = + sources + .filterNot( _.exists ) + .diff( Seq(defaultSourceDirectory) ) + assert( + nonExisting.isEmpty, + "Some sources do not exist: \n"+nonExisting.mkString("\n") + ) + } + assertSourceDirectories() + + + + + /** SBT-like dependency builder DSL */ + class GroupIdAndArtifactId( groupId: String, artifactId: String ){ + def %(version: String) = new MavenDependency(groupId, artifactId, version)(lib.logger) + } + implicit class DependencyBuilder(groupId: String){ + def %%(artifactId: String) = new GroupIdAndArtifactId( groupId, artifactId+"_"+scalaMajorVersion ) + def %(artifactId: String) = new GroupIdAndArtifactId( groupId, artifactId ) + } + + final def BuildDependency(path: String) = cbt.BuildDependency( + context.copy( + cwd = path, + args = Seq() + ) + ) + + def triggerLoopFiles: Seq[File] = sources ++ transitiveDependencies.collect{ case b: TriggerLoop => b.triggerLoopFiles }.flatten + + + def localJars : Seq[File] = + Seq(projectDirectory + "/lib/") + .map(new File(_)) + .filter(_.exists) + .flatMap(_.listFiles) + .filter(_.toString.endsWith(".jar")) + + //def cacheJar = false + override def dependencyClasspath : ClassPath = ClassPath(localJars) ++ super.dependencyClasspath + override def dependencyJars : Seq[File] = localJars ++ super.dependencyJars + + def exportedClasspath : ClassPath = ClassPath(Seq(compile)) + def exportedJars: Seq[File] = Seq() + // ========== compile, run, test ========== + + /** scalac options used for zinc and scaladoc */ + def scalacOptions: Seq[String] = Seq( "-feature", "-deprecation", "-unchecked" ) + + val updated: Boolean = { + val existingClassFiles = lib.listFilesRecursive(compileTarget) + val sourcesChanged = existingClassFiles.nonEmpty && { + val oldestClassFile = existingClassFiles.sortBy(_.lastModified).head + val oldestClassFileAge = oldestClassFile.lastModified + val changedSourceFiles = sourceFiles.filter(_.lastModified > oldestClassFileAge) + if(changedSourceFiles.nonEmpty){ + /* + println(changedSourceFiles) + println(changedSourceFiles.map(_.lastModified)) + println(changedSourceFiles.map(_.lastModified > oldestClassFileAge)) + println(oldestClassFile) + println(oldestClassFileAge) + println("-"*80) + */ + } + changedSourceFiles.nonEmpty + } + sourcesChanged || transitiveDependencies.map(_.updated).fold(false)(_ || _) + } + + private object cacheCompileBasicBuild extends Cache[File] + def compile: File = cacheCompileBasicBuild{ + //println(transitiveDependencies.filter(_.updated).mkString("\n")) + lib.compile( + updated, + sourceFiles, compileTarget, dependencyClasspath, scalacOptions, + zincVersion = zincVersion, scalaVersion = scalaVersion + ) + } + + def runClass: String = "Main" + def run: Unit = lib.runMainIfFound( runClass, Seq(), classLoader ) + + def test: Unit = lib.test(context) + + context.logger.composition(">"*80) + context.logger.composition("class "+this.getClass) + context.logger.composition("dir "+context.cwd) + context.logger.composition("sources "+sources.toList.mkString(" ")) + context.logger.composition("target "+target) + context.logger.composition("dependencyTree\n"+dependencyTree) + context.logger.composition("<"*80) + + // ========== cbt internals ========== + private[cbt] def finalBuild = this + override def show = this.getClass.getSimpleName + "("+context.cwd+")" +} diff --git a/stage2/Lib.scala b/stage2/Lib.scala new file mode 100644 index 0000000..b92e0b3 --- /dev/null +++ b/stage2/Lib.scala @@ -0,0 +1,436 @@ +package cbt +import cbt.paths._ + +import java.io._ +import java.net._ +import java.lang.reflect.InvocationTargetException +import java.nio.file.{Path =>_,_} +import java.nio.file.Files.readAllBytes +import java.security.MessageDigest +import java.util.jar._ + +import scala.collection.immutable.Seq +import scala.reflect.runtime.{universe => ru} +import scala.util._ + +import ammonite.ops.{cwd => _,_} + +case class Developer(id: String, name: String, timezone: String, url: URL) +case class License(name: String, url: URL) + +/** Don't extend. Create your own libs :). */ +final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ + lib => + + val buildClassName = "Build" + val buildBuildClassName = "BuildBuild" + + /** Loads Build for given Context */ + def loadDynamic(context: Context, default: Context => Build = new Build(_)): Build = { + context.logger.composition( context.logger.showInvocation("Build.loadDynamic",context) ) + loadRoot(context, default).finalBuild + } + /** + Loads whatever Build needs to be executed first in order to eventually build the build for the given context. + This can either the Build itself, of if exists a BuildBuild or a BuildBuild for a BuildBuild and so on. + */ + def loadRoot(context: Context, default: Context => Build = new Build(_)): Build = { + context.logger.composition( context.logger.showInvocation("Build.loadRoot",context) ) + def findStartDir(cwd: String): String = { + val buildDir = realpath(cwd+"/build") + if(new File(buildDir).exists) findStartDir(buildDir) else cwd + } + + val start = findStartDir(context.cwd) + + val useBasicBuildBuild = context.cwd == start + + val rootBuildClassName = if( useBasicBuildBuild ) buildBuildClassName else buildClassName + try{ + if(useBasicBuildBuild) default( context ) else new cbt.BuildBuild( context.copy( cwd = start ) ) + } catch { + case e:ClassNotFoundException if e.getMessage == rootBuildClassName => + throw new Exception(s"no class $rootBuildClassName found in "+start) + } + } + + def compile( + updated: Boolean, + sourceFiles: Seq[File], compileTarget: File, dependenyClasspath: ClassPath, + compileArgs: Seq[String], zincVersion: String, scalaVersion: String + ): File = { + if(sourceFiles.nonEmpty) + lib.zinc( + updated, sourceFiles, compileTarget, dependenyClasspath, compileArgs + )( zincVersion = zincVersion, scalaVersion = scalaVersion ) + compileTarget + } + + def srcJar(sources: Seq[File], artifactId: String, version: String, jarTarget: File): File = { + val file = new File(jarTarget+"/"+artifactId+"-"+version+"-sources.jar") + lib.jarFile(file, sources) + file + } + + def jar(artifactId: String, version: String, compileTarget: File, jarTarget: File): File = { + val file = new File(jarTarget+"/"+artifactId+"-"+version+".jar") + lib.jarFile(file, Seq(compileTarget)) + file + } + + def docJar( + sourceFiles: Seq[File], + dependenyClasspath: ClassPath, + apiTarget: File, + jarTarget: File, + artifactId: String, + version: String, + compileArgs: Seq[String] + ): File = { + class DisableSystemExit extends Exception + object DisableSystemExit{ + def apply(e: Throwable): Boolean = { + e match { + case i: InvocationTargetException => apply(i.getTargetException) + case _: DisableSystemExit => true + case _ => false + } + } + } + + // FIXME: get this dynamically somehow, or is this even needed? + val javacp = ClassPath( + "/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/cldrdata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/dnsns.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/jaccess.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/jfxrt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/localedata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/nashorn.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/sunec.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/sunjce_provider.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/sunpkcs11.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/zipfs.jar:/System/Library/Java/Extensions/MRJToolkit.jar".split(":").toVector.map(new File(_)) + ) + + mkdir(Path(apiTarget)) + if(sourceFiles.nonEmpty){ + System.err.println("creating docs") + try{ + System.setSecurityManager( + new SecurityManager{ + override def checkPermission( permission: java.security.Permission ) = { + if( permission.getName.startsWith("exitVM") ) throw new DisableSystemExit + } + } + ) + redirectOutToErr{ + runMain( + "scala.tools.nsc.ScalaDoc", + Seq( + // FIXME: can we use compiler dependency here? + "-cp", /*javacp+":"+*/ScalaDependencies(logger).classpath.string + ":" + dependenyClasspath.string, + "-d", apiTarget.toString + ) ++ compileArgs ++ sourceFiles.map(_.toString), + new URLClassLoader( + ScalaDependencies(logger).classpath ++ javacp, + ClassLoader.getSystemClassLoader + ) + ) + } + } catch { + case e:InvocationTargetException if DisableSystemExit(e) => + } finally { + System.setSecurityManager(null) + } + } + val docJar = new File(jarTarget+"/"+artifactId+"-"+version+"-javadoc.jar") + lib.jarFile(docJar, Vector(apiTarget)) + docJar + } + + def test( context: Context ) = { + logger.lib(s"invoke testDefault( $context )") + loadDynamic( + context.copy( cwd = context.cwd+"/test/" ), + new Build(_) with mixins.Test + ).run + logger.lib(s"return testDefault( $context )") + + } + + // task reflection helpers + import ru._ + private lazy val anyRefMembers = ru.typeOf[AnyRef].members.toVector.map(taskName) + def taskNames(tpe: Type) = tpe.members.toVector.flatMap(lib.toTask).map(taskName).sorted + private def taskName(method: Symbol) = method.name.decodedName.toString + def toTask(symbol: Symbol): Option[MethodSymbol] = { + Option(symbol) + .filter(_.isPublic) + .filter(_.isMethod) + .map(_.asMethod) + .filter(_.paramLists.flatten.size == 0) + .filterNot(taskName(_) contains "$") + .filterNot(t => anyRefMembers contains taskName(t)) + } + + class ReflectBuild(build: Build) extends ReflectObject(build){ + def usage = { + val baseTasks = lib.taskNames(ru.typeOf[Build]) + val thisTasks = lib.taskNames(subclassType) diff baseTasks + ( + ( + if( thisTasks.nonEmpty ){ + s"""Methods provided by Build ${build.context.cwd} + + ${thisTasks.mkString(" ")} + +""" + } else "" + ) + s"""Methods provided by CBT (but possibly overwritten) + + ${baseTasks.mkString(" ")}""" + ) + "\n" + } + } + + abstract class ReflectObject[T:scala.reflect.ClassTag](obj: T){ + lazy val mirror = ru.runtimeMirror(obj.getClass.getClassLoader) + lazy val subclassType = mirror.classSymbol(obj.getClass).toType + def usage: String + def callNullary( taskName: Option[String] ): Unit = { + taskName + .map{ n => subclassType.member(ru.TermName(n).encodedName) } + .filter(_ != ru.NoSymbol) + .flatMap(toTask _) + .map{ methodSymbol => + val result = mirror.reflect(obj).reflectMethod(methodSymbol)() + // Try to render console representation. Probably not the best way to do this. + val method = scala.util.Try( result.getClass.getDeclaredMethod("toConsole") ) + + method.foreach(m => println(m.invoke(result))) + method.recover{ + case e:NoSuchMethodException if e.getMessage contains "toConsole" => + result match { + case () => "" + case other => println( other.toString ) // no method .toConsole, using to String + } + } + }.getOrElse{ + taskName.foreach{ n => + System.err.println(s"Method not found: $n") + System.err.println("") + } + System.err.println(usage) + System.exit(1) + } + } + } + + + // file system helpers + def basename(path: String) = path.stripSuffix("/").split("/").last + def basename(path: File) = path.toString.stripSuffix("/").split("/").last + def dirname(path: String) = realpath(path).stripSuffix("/").split("/").dropRight(1).mkString("/") + def realpath(name: String) = Paths.get(new File(name).getAbsolutePath).normalize.toString + def realpath(name: File) = new File(Paths.get(name.getAbsolutePath).normalize.toString) + def nameAndContents(file: File) = basename(file.toString) -> readAllBytes(Paths.get(file.toString)) + + def jarFile( jarFile: File, files: Seq[File] ): Unit = { + logger.lib("Start packaging "+jarFile) + val manifest = new Manifest + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0") + val jar = new JarOutputStream(new FileOutputStream(jarFile.toString), manifest) + + val names = for { + base <- files.filter(_.exists).map(realpath) + file <- listFilesRecursive(base) if file.isFile + } yield { + val name = if(base.isDirectory){ + file.toString stripPrefix base.toString + } else file.toString + val entry = new JarEntry( name ) + entry.setTime(file.lastModified) + jar.putNextEntry(entry) + jar.write( Files.readAllBytes( Paths.get(file.toString) ) ) + jar.closeEntry + name + } + + val duplicateFiles = (names diff names.distinct).distinct + assert( + duplicateFiles.isEmpty, + s"Conflicting file names when trying to create $jarFile: "+duplicateFiles.mkString(", ") + ) + + jar.close + logger.lib("Done packaging "+jarFile) + } + + lazy val passphrase = + Option(System.console).getOrElse( + throw new Exception("Can't access Console. This probably shouldn't be run through Nailgun.") + ).readPassword( + "GPG Passphrase please:" + ).mkString + + def sign(file: File): File = { + //http://stackoverflow.com/questions/16662408/correct-way-to-sign-and-verify-signature-using-bouncycastle + val statusCode = + new ProcessBuilder( "gpg", "--batch", "--yes", "-a", "-b", "-s", "--passphrase", passphrase, file.toString ) + .inheritIO.start.waitFor + + if( 0 != statusCode ) throw new Exception("gpg exited with status code "+statusCode) + + new File(file+".asc") + } + + //def requiredForPom[T](name: String): T = throw new Exception(s"You need to override `def $name` in order to generate a valid pom.") + + def pom( + groupId: String, + artifactId: String, + version: String, + name: String, + description: String, + url: URL, + developers: Seq[Developer], + licenses: Seq[License], + scmUrl: String, // seems like invalid URLs are used here in pom files + scmConnection: String, + dependencies: Seq[Dependency], + pomExtra: Seq[scala.xml.Node], + jarTarget: File + ): File = { + val xml = + <project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"> + <modelVersion>4.0.0</modelVersion> + <groupId>{groupId}</groupId> + <artifactId>{artifactId}</artifactId> + <version>{version}</version> + <packaging>jar</packaging> + <name>{name}</name> + <description>{description}</description> + <url>{url}</url> + <licenses> + {licenses.map{ license => + <license> + <name>{license.name}</name> + <url>{license.url}</url> + <distribution>repo</distribution> + </license> + }} + </licenses> + <developers> + {developers.map{ developer => + <developer> + <id>{developer.id}</id> + <name>{developer.name}</name> + <timezone>{developer.timezone}</timezone> + <url>{developer.url}</url> + </developer> + }} + </developers> + <scm> + <url>{scmUrl}</url> + <connection>{scmConnection}</connection> + </scm> + {pomExtra} + <dependencies> + { + dependencies.map{ + case d:ArtifactInfo => + <dependency> + <groupId>{d.groupId}</groupId> + <artifactId>{d.artifactId}</artifactId> + <version>{d.version}</version> + </dependency> + } + } + </dependencies> + </project> + val path = new File(jarTarget+"/"+artifactId+"-"+version+".pom") + write.over(Path(path), "<?xml version='1.0' encoding='UTF-8'?>\n" + xml.toString) + path + } + + def concurrently[T,R]( concurrencyEnabled: Boolean )( items: Seq[T] )( projection: T => R ): Seq[R] = { + if(concurrencyEnabled) items.par.map(projection).seq + else items.map(projection) + } + + def publishSnapshot( sourceFiles: Seq[File], artifacts: Seq[File], url: URL ): Unit = { + if(sourceFiles.nonEmpty){ + val files = artifacts.map(nameAndContents) + uploadAll(url, files) + } + } + + def publishSigned( sourceFiles: Seq[File], artifacts: Seq[File], url: URL ): Unit = { + // TODO: make concurrency configurable here + if(sourceFiles.nonEmpty){ + val files = (artifacts ++ artifacts.map(sign)).map(nameAndContents) + lazy val checksums = files.flatMap{ + case (name, content) => Seq( + name+".md5" -> md5(content).toArray.map(_.toByte), + name+".sha1" -> sha1(content).toArray.map(_.toByte) + ) + } + val all = (files ++ checksums) + uploadAll(url, all) + } + } + + + def uploadAll(url: URL, nameAndContents: Seq[(String, Array[Byte])]): Unit = + nameAndContents.map{ case(name, content) => upload(name, content, url) } + + def upload(fileName: String, fileContents: Array[Byte], baseUrl: URL): Unit = { + import java.net._ + import java.io._ + logger.task("uploading "+fileName) + val url = new URL( + baseUrl + fileName + ) + val httpCon = url.openConnection.asInstanceOf[HttpURLConnection] + httpCon.setDoOutput(true) + httpCon.setRequestMethod("PUT") + val userPassword = read(Path(sonatypeLogin)).trim + val encoding = new sun.misc.BASE64Encoder().encode(userPassword.getBytes) + httpCon.setRequestProperty("Authorization", "Basic " + encoding) + httpCon.setRequestProperty("Content-Type", "application/binary") + httpCon.getOutputStream.write( + fileContents + ) + httpCon.getInputStream + } + + + // code for continuous compile + def watch(files: Seq[File])(action: PartialFunction[File, Unit]): Unit = { + import com.barbarysoftware.watchservice._ + import scala.collection.JavaConversions._ + val watcher = WatchService.newWatchService + + files.map{ + file => + if(file.isFile) new File( dirname(file.toString) ) + else file + }.distinct.map{ file => + val watchableFile = new WatchableFile(file) + val key = watchableFile.register( + watcher, + StandardWatchEventKind.ENTRY_CREATE, + StandardWatchEventKind.ENTRY_DELETE, + StandardWatchEventKind.ENTRY_MODIFY + ) + } + + scala.util.control.Breaks.breakable{ + while(true){ + logger.loop("Waiting for file changes...") + Option(watcher.take).map{ + key => + val changedFiles = key + .pollEvents + .filterNot(_.kind == StandardWatchEventKind.OVERFLOW) + .map(_.context.toString) + .map(new File(_)) + changedFiles.foreach( f => logger.loop("Changed: "+f) ) + changedFiles.collect(action) + key.reset + } + } + } + } +} diff --git a/stage2/Scaffold.scala b/stage2/Scaffold.scala new file mode 100644 index 0000000..00c8706 --- /dev/null +++ b/stage2/Scaffold.scala @@ -0,0 +1,148 @@ +package cbt +import java.io._ +import java.net._ +import ammonite.ops.{cwd => _,_} + +trait Scaffold{ + def logger: Logger + + def generateBasicBuildFile( + projectDirectory: File, + scalaVersion: String, + groupId: String, + artifactId: String, + version: String + ): Unit = { + /** + TODO: + - make behavior more user friendly: + - not generate half and then throw exception for one thing already existing + - maybe not generate all of this, e.g. offer different variants + */ + + val generatedFiles = Seq( + "build/build.scala" -> s"""import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +class Build(context: Context) extends BasicBuild(context) with BuildShared{ + override def artifactId: String = "$artifactId" + override def groupId = "$groupId" + + override def dependencies = super.dependencies ++ Seq( // don't forget super.dependencies here + // "org.cvogt" %% "scala-extensions" % "0.4.1" + ) + + // required for .pom file + override def name = artifactId + override def description : String = lib.requiredForPom("description") +} +""", + + "build/build/build.scala" -> s"""import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +class Build(context: Context) extends BuildBuild(context){ + override def scalaVersion: String = "2.11.7" + + override def dependencies = super.dependencies ++ Seq( + BuildDependency( projectDirectory + "/../build-shared/") + // , "com.lihaoyi" %% "ammonite-ops" % "0.5.5" + ) +} +""", + + "test/Main.scala" -> s"""object Main{ + def main( args: Array[String] ) = { + assert( false, "Go. Write some tests :)!" ) + } +} +""", + + "test/build/build.scala" -> s"""import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +class Build(context: cbt.Context) extends BasicBuild(context) with BuildShared/* with cbt.mixins.ScalaTest*/{ + // def scalaTestVersion = "2.2.6" + + override def dependencies = super.dependencies ++ Seq( + // , "org.scalacheck" %% "scalacheck" % "1.13.0" + ) +} +""", + + "test/build/build/build.scala" -> s"""import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +class Build(context: Context) extends BuildBuild(context){ + override def scalaVersion: String = "2.11.7" + + override def dependencies = super.dependencies ++ Seq( + BuildDependency( projectDirectory + "/../../build-shared/") + // , "com.lihaoyi" %% "ammonite-ops" % "0.5.5" + ) +} +""", + + "build-shared/build/build.scala" -> s"""import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +class Build(context: Context) extends BasicBuild(context){ + override def scalaVersion: String = "$scalaVersion" + + override def dependencies = super.dependencies ++ Seq( // don't forget super.dependencies here + CbtDependency + // , "org.cvogt" %% "scala-extensions" % "0.4.1" + ) +} +""", + + "build-shared/BuildShared.scala" -> s"""import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +trait BuildShared extends BasicBuild{ + override def scalaVersion: String = "$scalaVersion" + override def enableConcurrency = false // enable for speed, disable for debugging + + override def groupId = "$groupId" + override def version = "$version" + + // required for .pom file + override def url : URL = lib.requiredForPom("url") + override def developers: Seq[Developer] = lib.requiredForPom("developers") + override def licenses : Seq[License] = lib.requiredForPom("licenses") + override def scmUrl : String = lib.requiredForPom("scmUrl") + override def scmConnection: String = lib.requiredForPom("scmConnection") + override def pomExtra: Seq[scala.xml.Node] = Seq() +} +""" + ) + + generatedFiles.map{ + case ( fileName, code ) => + scala.util.Try{ + write( Path(projectDirectory+"/"+fileName), code ) + import scala.Console._ + println( GREEN + "Created " + fileName + RESET ) + } + }.foreach( + _.recover{ + case e: java.nio.file.FileAlreadyExistsException => + e.printStackTrace + }.get + ) + return () + } + +}
\ No newline at end of file diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala new file mode 100644 index 0000000..05b7c58 --- /dev/null +++ b/stage2/Stage2.scala @@ -0,0 +1,47 @@ +package cbt +import cbt.paths._ +import java.io._ +import scala.collection.immutable.Seq + +object Stage2{ + def main(args: Array[String]) = { + import java.time.LocalTime.now + val init = new Stage1.Init(args) + import java.time._ + val start = LocalTime.now() + def timeTaken = Duration.between(start, LocalTime.now()).toMillis + init.logger.stage2(s"[$now] Stage2 start") + + import init._ + val loop = argsV.lift(1) == Some("loop") + val direct = argsV.lift(1) == Some("direct") + val taskIndex = if(loop || direct) 2 else 1 + val task = argsV.lift( taskIndex ) + + val lib = new Lib(new Stage1.Init(args).logger) + + val context = Context( cwd, argsV.drop( taskIndex + 1 ), logger ) + val first = lib.loadRoot( context ) + val build = first.finalBuild + + val res = if( loop ){ + // TODO: this should allow looping over task specific files, like test files as well + val triggerFiles = first.triggerLoopFiles.map(lib.realpath) + val triggerCbtFiles = Seq( nailgun, stage1, stage2 ).map(lib.realpath _) + val allTriggerFiles = triggerFiles ++ triggerCbtFiles + + logger.loop("Looping change detection over:\n - "+allTriggerFiles.mkString("\n - ")) + + lib.watch(allTriggerFiles){ + case file if triggerCbtFiles.exists(file.toString startsWith _.toString) => + logger.loop("Change is in CBT' own source code.") + logger.loop("Restarting CBT.") + scala.util.control.Breaks.break + case file if triggerFiles.exists(file.toString startsWith _.toString) => + new lib.ReflectBuild( lib.loadDynamic(context) ).callNullary(task) + } + } else new lib.ReflectBuild(build).callNullary(task) + init.logger.stage2(s"[$now] Stage2 end") + res + } +} diff --git a/stage2/dependencies.scala b/stage2/dependencies.scala new file mode 100644 index 0000000..8ed36eb --- /dev/null +++ b/stage2/dependencies.scala @@ -0,0 +1,36 @@ +package cbt +import java.io.File +import scala.collection.immutable.Seq +/* +sealed abstract class ProjectProxy extends Ha{ + protected def delegate: ProjectMetaData + def artifactId: String = delegate.artifactId + def groupId: String = delegate.groupId + def version: String = delegate.version + def exportedClasspath = delegate.exportedClasspath + def dependencies = Seq(delegate) +} +*/ +trait TriggerLoop extends Dependency{ + def triggerLoopFiles: Seq[File] +} +/** You likely want to use the factory method in the BasicBuild class instead of this. */ +case class BuildDependency(context: Context) extends TriggerLoop{ + override def show = this.getClass.getSimpleName + "("+context.cwd+")" + final override lazy val logger = context.logger + final override lazy val lib: Lib = new Lib(logger) + private val root = lib.loadRoot( context.copy(args=Seq()) ) + lazy val build = root.finalBuild + def exportedClasspath = ClassPath(Seq()) + def exportedJars = Seq() + def dependencies = Seq(build) + def triggerLoopFiles = root.triggerLoopFiles + final val updated = build.updated +} +/* +case class DependencyOr(first: BuildDependency, second: MavenDependency) extends ProjectProxy with BuildDependencyBase{ + val isFirst = new File(first.context.cwd).exists + def triggerLoopFiles = if(isFirst) first.triggerLoopFiles else Seq() + protected val delegate = if(isFirst) first else second +} +*/
\ No newline at end of file diff --git a/stage2/mixins.scala b/stage2/mixins.scala new file mode 100644 index 0000000..5ed26d8 --- /dev/null +++ b/stage2/mixins.scala @@ -0,0 +1,35 @@ +package cbt +package mixins +import scala.collection.immutable.Seq +import java.io._ +trait Test extends Build{ + lazy val testedBuild = BuildDependency(projectDirectory+"/../") + override def dependencies = Seq( testedBuild ) ++ super.dependencies + override def scalaVersion = testedBuild.build.scalaVersion +} +trait Sbt extends Build{ + override def sources = Seq(new File(projectDirectory+"/src/main/scala/")) +} +trait SbtTest extends Test{ + override def sources = Vector(new File(projectDirectory+"/../src/test/scala")) +} +trait ScalaTest extends Build with Test{ + def scalaTestVersion: String + + override def dependencies = Seq( + "org.scalatest" %% "scalatest" % scalaTestVersion + ) ++ super.dependencies + + // workaround probable ScalaTest bug throwing away the outer classloader. Not caching doesn't nest them. + override def cacheDependencyClassLoader = false + + override def run = { + val discoveryPath = compile.toString+"/" + context.logger.lib("discoveryPath: "+discoveryPath) + lib.runMain( + "org.scalatest.tools.Runner", + Seq("-R", discoveryPath, "-oF") ++ context.args.drop(1), + classLoader + ) + } +} diff --git a/test/build/build.scala b/test/build/build.scala new file mode 100644 index 0000000..92a964b --- /dev/null +++ b/test/build/build.scala @@ -0,0 +1,5 @@ +import scala.collection.immutable.Seq +import java.io.File +class Build(context: cbt.Context) extends cbt.Build(context){ + override def dependencies = Seq( cbt.CbtDependency(context.logger) ) ++ super.dependencies +} diff --git a/test/nothing/placeholder b/test/nothing/placeholder new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/nothing/placeholder diff --git a/test/test.scala b/test/test.scala new file mode 100644 index 0000000..d684744 --- /dev/null +++ b/test/test.scala @@ -0,0 +1,77 @@ +import cbt.paths._ +import scala.collection.immutable.Seq + +object Main{ + // micro framework + var successes = 0 + var failures = 0 + def assert(condition: Boolean, msg: String = null) = { + scala.util.Try{ + Predef.assert(condition, msg) + }.map{ _ => + print(".") + successes += 1 + }.recover{ + case e: AssertionError => + println("FAILED") + e.printStackTrace + failures += 1 + }.get + } + + def runCbt(path: String, args: Seq[String]) = { + import java.io._ + val allArgs = ((cbtHome + "/cbt") +: args) + val pb = new ProcessBuilder( allArgs :_* ) + pb.directory(new File(cbtHome + "/test/" + path)) + val p = pb.start + val berr = new BufferedReader(new InputStreamReader(p.getErrorStream)); + val bout = new BufferedReader(new InputStreamReader(p.getInputStream)); + p.waitFor + import collection.JavaConversions._ + val err = Stream.continually(berr.readLine()).takeWhile(_ != null).mkString("\n") + val out = Stream.continually(bout.readLine()).takeWhile(_ != null).mkString("\n") + Result(out, err, p.exitValue == 0) + } + case class Result(out: String, err: String, exit0: Boolean) + def assertSuccess(res: Result) = { + assert(res.exit0,res.toString) + } + + // tests + def usage(path: String) = { + val usageString = "Methods provided by CBT" + val res = runCbt(path, Seq()) + assert(res.out == "", res.out) + assert(res.err contains usageString, res.err) + } + def compile(path: String) = { + val res = runCbt(path, Seq("compile")) + assertSuccess(res) + // assert(res.err == "", res.err) // FIXME: enable this + } + def main(args: Array[String]): Unit = { + import cbt._ + + println("Running tests ") + + usage("nothing") + compile("nothing") + + { + val logger = new Logger(Set[String]()) + val noContext = Context(cbtHome + "/test/" + "nothing",Seq(),logger) + val b = new Build(noContext){ + override def dependencies = Seq( + MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0")(logger), + MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0")(logger) + ) + } + val cp = b.classpath + assert(cp.strings.distinct == cp.strings, "duplicates in classpath: "+cp) + } + + println(" DONE!") + println(successes+" succeeded, "+ failures+" failed" ) + } +} |