From faa4110fc7d8b92cac1e50a6952cdf3bc85d1cdf Mon Sep 17 00:00:00 2001 From: Stefan Zeiger Date: Tue, 19 Jul 2016 15:14:34 +0200 Subject: Switch Windows CI build to sbt (w/ some sbt build improvements) - Use sbt in `integrate/windows`: This essentially combines the Unix CI jobs `validate/publish-core` and `validate/test`, first publishing a local release built with STARR and then building a new version with that and running all tests on it. - Unify repository handling across build scripts: A new function `generateRepositoriesConfig` in `common`, based on the existing code in `integrate/bootstrap`, writes the `repositories` file for sbt, either with or without an extra bootstrap repository for resolving a previously built version. It is used in all CI scripts to ensure that artifacts are only resolved through the sanctioned proxies and upstream repositories. - The repository URL arguments in `setupPublishCore` and `setupValidateTest` are now optional as well. These commands are used without a URL from `integrate/windows`, which publishes to `local` instead of a temporary remote repository. - `testAll` is now a task instead of a command. It runs the same sequence of sub-tasks as before but does not propagate failures immediately. It always runs all subtasks and reports errors at the end. - The `generateBuildCharacterPropertiesFile` task now includes all properties from `versions.properties` (whose values have potentially been overwritten with `-D` options) in `buildcharacter.properties`. --- scripts/common | 20 ++++++++++++++++++++ scripts/jobs/integrate/bootstrap | 24 ++++-------------------- scripts/jobs/integrate/windows | 22 +++++++++++++--------- scripts/jobs/validate/publish-core | 7 +++++-- scripts/jobs/validate/test | 7 +++++-- 5 files changed, 47 insertions(+), 33 deletions(-) (limited to 'scripts') diff --git a/scripts/common b/scripts/common index bfddf3d149..95389e5495 100644 --- a/scripts/common +++ b/scripts/common @@ -154,3 +154,23 @@ EOF fi popd } + +# Generate a repositories file with all allowed repositories in our build environment. +# Takes one optional argument, the private repository URL. +# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html +function generateRepositoriesConfig() { + jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} + sbtRepositoryConfig="$scriptsDir/sbt-repositories-config" + echo > "$sbtRepositoryConfig" '[repositories]' + if [ -n "$1" ] + then + echo >> "$sbtRepositoryConfig" " private-repo: $1" + fi + cat >> "$sbtRepositoryConfig" << EOF + jcenter-cache: $jcenterCacheUrl + typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly + sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + maven-central + local +EOF +} diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap index bdc50323a8..4d5dae89a2 100644 --- a/scripts/jobs/integrate/bootstrap +++ b/scripts/jobs/integrate/bootstrap @@ -85,10 +85,8 @@ moduleVersioning=${moduleVersioning-"versions.properties"} publishPrivateTask=${publishPrivateTask-"publish"} -publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"} -publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"} -publishStarrPrivateTask=${publishStarrPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built STARR before) -publishLockerPrivateTask=${publishLockerPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built locker before) +publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"} +publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"} forceRebuild=${forceRebuild-no} @@ -111,21 +109,7 @@ mkdir -p $baseDir/resolutionScratch_ # repo used to publish "locker" scala to (to start the bootstrap) releaseTempRepoCred="private-repo" releaseTempRepoUrl=${releaseTempRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-release-temp/"} -jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} - -# Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs, -# so we need to configure SBT to use these rather than its default, Maven Central. -# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html -sbtRepositoryConfig="$scriptsDir/repositories-scala-release" -cat > "$sbtRepositoryConfig" << EOF -[repositories] - private-repo: $releaseTempRepoUrl - jcenter-cache: $jcenterCacheUrl - typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly - sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] - maven-central - local -EOF +generateRepositoriesConfig $releaseTempRepoUrl ##### git gfxd() { @@ -575,7 +559,7 @@ publishSonatype() { -Dstarr.version=$SCALA_VER \ ${updatedModuleVersions[@]} \ "setupBootstrapPublish $releaseTempRepoUrl $SCALA_VER" \ - publishSigned | grep -v "was too long to be displayed in the webview, and will be left out" + $publishSonatypeTaskCore | grep -v "was too long to be displayed in the webview, and will be left out" echo "### Publishing modules to sonatype" # build/test/publish scala core modules to sonatype (this will start a new staging repo) diff --git a/scripts/jobs/integrate/windows b/scripts/jobs/integrate/windows index ba48c5bc25..5e04b0b380 100755 --- a/scripts/jobs/integrate/windows +++ b/scripts/jobs/integrate/windows @@ -1,15 +1,19 @@ #!/bin/bash -./pull-binary-libs.sh - -export ANT_OPTS="-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M" - -# TODO: don't hardcode these paths -- should be in scala/scala-jenkins-infra, passed in through env vars from jenkins -export PATH='/cygdrive/c/Program Files/Java/jdk1.8.0_92/bin:/cygdrive/c/apache-ant-1.9.6/bin:/cygdrive/c/Program Files (x86)/Git-2.5.3/Cmd:/bin:/usr/bin:' -export JAVA_HOME='C:/Program Files/Java/jdk1.8.0_92' +baseDir=${WORKSPACE-`pwd`} +scriptsDir="$baseDir/scripts" +. $scriptsDir/common java -version javac -version -ant -version -ant test-opt +generateRepositoriesConfig + +SBT="java $JAVA_OPTS -Dsbt.ivy.home=$WORKSPACE/.ivy2 -jar $sbtLauncher -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig" + +# Build locker with STARR +$SBT --warn "setupPublishCore" generateBuildCharacterPropertiesFile publishLocal + +# Build quick and run the tests +parseScalaProperties buildcharacter.properties +$SBT -Dstarr.version=$maven_version_number --warn "setupValidateTest" testAll | grep -v "was too long to be displayed in the webview, and will be left out" diff --git a/scripts/jobs/validate/publish-core b/scripts/jobs/validate/publish-core index b0bfd48083..c71fbd12b7 100755 --- a/scripts/jobs/validate/publish-core +++ b/scripts/jobs/validate/publish-core @@ -9,6 +9,9 @@ baseDir=${WORKSPACE-`pwd`} scriptsDir="$baseDir/scripts" . $scriptsDir/common +generateRepositoriesConfig $prRepoUrl +SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig" + case $prDryRun in yep) echo "DRY RUN" @@ -16,7 +19,7 @@ case $prDryRun in ;; *) echo ">>> Getting Scala version number." - $SBT_CMD --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile + $SBT --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile parseScalaProperties buildcharacter.properties # produce maven_version_number echo ">>> Checking availability of Scala ${maven_version_number} in $prRepoUrl." @@ -27,7 +30,7 @@ case $prDryRun in if $libraryAvailable && $reflectAvailable && $compilerAvailable; then echo "Scala core already built!" else - $SBT_CMD --warn "setupPublishCore $prRepoUrl" publish + $SBT --warn "setupPublishCore $prRepoUrl" publish fi mv buildcharacter.properties jenkins.properties # parsed by the jenkins job diff --git a/scripts/jobs/validate/test b/scripts/jobs/validate/test index dd36c5db18..9938319dd8 100755 --- a/scripts/jobs/validate/test +++ b/scripts/jobs/validate/test @@ -4,6 +4,9 @@ baseDir=${WORKSPACE-`pwd`} scriptsDir="$baseDir/scripts" . $scriptsDir/common +generateRepositoriesConfig $prRepoUrl +SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig" + case $prDryRun in yep) @@ -15,12 +18,12 @@ case $prDryRun in # build quick using STARR built upstream, as specified by scalaVersion # (in that sense it's locker, since it was built with starr by that upstream job); # and run JUnit tests, partest, OSGi tests, MiMa and scaladoc - $SBT_CMD \ + $SBT \ -Dstarr.version=$scalaVersion \ --warn \ "setupValidateTest $prRepoUrl" \ $testExtraArgs \ - testAll + testAll | grep -v "was too long to be displayed in the webview, and will be left out" ;; -- cgit v1.2.3