From 3410d25e37820e8936acc3fc1a444069dc415524 Mon Sep 17 00:00:00 2001 From: Adriaan Moors Date: Tue, 23 Dec 2014 15:54:18 -0800 Subject: Towards travis-style CI Move script to the repo it acts on, out of scala/jenkins-scripts Had to make some changes: - scala checkout is now in root ($WORKSPACE) - credentials are actually in homedir (sonatype/private-repo) - inline pr-scala-common - don't run tests for actors-migration due to timeout (TODO: revert?) --- scripts/jobs/scala-release-2.11.x-build | 117 ++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 30 deletions(-) (limited to 'scripts/jobs') diff --git a/scripts/jobs/scala-release-2.11.x-build b/scripts/jobs/scala-release-2.11.x-build index 141ccb7930..21fbb8fa76 100755 --- a/scripts/jobs/scala-release-2.11.x-build +++ b/scripts/jobs/scala-release-2.11.x-build @@ -8,12 +8,12 @@ # jenkinside # # -# - ~/.ivy2/.credentials (for sonatype) +# - ~/.credentials (for sonatype) # realm=Sonatype Nexus Repository Manager # host=oss.sonatype.org # user=lamp # password= -# - ~/.ivy2/.credentials-private-repo for private-repo.typesafe.com, as follows: +# - ~/.credentials-private-repo for private-repo.typesafe.com, as follows: # realm=Artifactory Realm # host=private-repo.typesafe.com # user=jenkinside @@ -46,7 +46,7 @@ # Rebuild modules with this Scala build, and stage them on sonatype as well # This script can be run in multiple modes. It is design to work without any input, # so that it could be run in Travis CI. In that mode, it'll build a release when -# the current HEAD of $SCALA_REF is tagged, and stage to sonatype. Otherwise, +# the current HEAD of the checkout in $WORKSPACE is tagged, and stage to sonatype. Otherwise, # it'll build a nightly. # # Since the nightlies are intended to be a drop in replacement, all modules are built with the @@ -56,18 +56,17 @@ # The other way to trigger a release is by setting the SCALA_VER_BASE env var. # # By default, we build the versions of the modules as specified by versions.properties -# (as specified in the $SCALA_REF commit). Set moduleVersioning to something random +# (as specified in the HEAD commit). Set moduleVersioning to something random # to trigger building HEAD of each module, generating a fresh -$(git describe)-nightly version for each. # # PS: set publishToSonatype to anything but "yes" to avoid publishing to sonatype -# (publishing only done when $SCALA_REF's HEAD is tagged / SCALA_VER_BASE is set.) +# (publishing only done when $WORKSPACE checkout's HEAD is tagged / SCALA_VER_BASE is set.) -SCALA_REF=${SCALA_REF-"2.11.x"} # set to something besides the default to build nightly snapshots of the modules instead of some tagged version moduleVersioning=${moduleVersioning-"versions.properties"} -baseDir=${baseDir-`pwd`} +baseDir=${WORKSPACE-`pwd`} publishPrivateTask=${publishPrivateTask-"publish"} publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"} publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"} @@ -86,12 +85,70 @@ forceRebuild=${forceRebuild-no} # overridden to "no" when no SCALA_VER_BASE is passed and HEAD is not tagged with a valid version tag # -antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check +antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak) clean="clean" # TESTING leave empty to speed up testing -scriptsDir="$( cd "$( dirname "$0" )/.." && pwd )" -. $scriptsDir/common -. $scriptsDir/pr-scala-common +scriptsDir="$WORKSPACE/scripts" + +# This is for forcibly stopping the job from a subshell (see test +# below). +trap "exit 1" TERM +export TOP_PID=$$ +set -e + +# Known problems : does not fare well with interrupted, partial +# compilations. We should perhaps have a multi-dependency version +# of do_i_have below + +LOGGINGDIR="$WORKSPACE/logs" +mkdir -p $LOGGINGDIR + +unset SBT_HOME +SBT_HOME="$WORKSPACE/.sbt" +mkdir -p $SBT_HOME +IVY_CACHE="$WORKSPACE/.ivy2" +mkdir -p $IVY_CACHE +rm -rf $IVY_CACHE/cache/org.scala-lang + +# temp dir where all 'non-build' operation are performed +TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX) +TMP_DIR="${TMP_ROOT_DIR}/tmp" +mkdir "${TMP_DIR}" + + +# detect sed version and how to enable extended regexes +SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)" + + + +# :docstring test: +# Usage: test +# Executes , logging the launch of the command to the +# main log file, and kills global script execution with the TERM +# signal if the commands ends up failing. +# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES, +# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST +# :end docstring: + +function test() { + echo "### $@" + "$@" + status=$? + if [ $status -ne 0 ]; then + say "### ERROR with $1" + kill -s TERM $TOP_PID + fi +} + +# :docstring say: +# Usage: say +# Prints to both console and the main log file. +# :end docstring: + +function say(){ + (echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log +} + # we must change ivy home to get a fresh ivy cache, otherwise we get half-bootstrapped scala # rm it in case it existed (and there's no ivy2-shadow, which indicates we're running in a TESTING environment)... @@ -102,15 +159,12 @@ mkdir -p $baseDir/ivy2 rm -rf $baseDir/resolutionScratch_ mkdir -p $baseDir/resolutionScratch_ -mkdir -p $baseDir/logs - - # repo used to publish "locker" scala to (to start the bootstrap) privateCred="private-repo" privateRepo="http://private-repo.typesafe.com/typesafe/scala-release-temp/" function parseScalaProperties(){ - propFile="$baseDir/scala/$1" + propFile="$baseDir/$1" if [ ! -f $propFile ]; then echo "Property file $propFile not found." exit 1 @@ -206,7 +260,7 @@ buildXML() { then echo "Found scala-xml $XML_VER; not building." else update scala scala-xml "$XML_REF" && gfxd - sbtBuild 'set version := "'$XML_VER'-DOC"' $clean doc 'set version := "'$XML_VER'"' "${buildTasks[@]}" + sbtBuild 'set version := "'$XML_VER'-DOC"' $clean doc 'set version := "'$XML_VER'"' test "${buildTasks[@]}" XML_BUILT="yes" # ensure the module is built and published when buildXML is invoked for the second time, see comment above fi } @@ -216,7 +270,7 @@ buildParsers() { then echo "Found scala-parser-combinators $PARSERS_VER; not building." else update scala scala-parser-combinators "$PARSERS_REF" && gfxd - sbtBuild 'set version := "'$PARSERS_VER'-DOC"' $clean doc 'set version := "'$PARSERS_VER'"' "${buildTasks[@]}" + sbtBuild 'set version := "'$PARSERS_VER'-DOC"' $clean doc 'set version := "'$PARSERS_VER'"' test "${buildTasks[@]}" PARSERS_BUILT="yes" fi } @@ -226,7 +280,7 @@ buildPartest() { then echo "Found scala-partest $PARTEST_VER; not building." else update scala scala-partest "$PARTEST_REF" && gfxd - sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' 'set VersionKeys.scalaCheckVersion := "'$SCALACHECK_VER'"' $clean "${buildTasks[@]}" + sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' 'set VersionKeys.scalaCheckVersion := "'$SCALACHECK_VER'"' $clean test "${buildTasks[@]}" PARTEST_BUILT="yes" fi } @@ -247,7 +301,7 @@ buildContinuations() { update scala scala-continuations $CONTINUATIONS_REF && gfxd $sbtCmd $sbtArgs 'project plugin' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ - 'set version := "'$CONTINUATIONS_VER'"' $clean "compile:package" "${buildTasks[@]}" # https://github.com/scala/scala-continuations/pull/4 + 'set version := "'$CONTINUATIONS_VER'"' $clean "compile:package" test "${buildTasks[@]}" # https://github.com/scala/scala-continuations/pull/4 CONT_PLUG_BUILT="yes" fi @@ -256,7 +310,7 @@ buildContinuations() { else update scala scala-continuations $CONTINUATIONS_REF && gfxd $sbtCmd $sbtArgs 'project library' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ - 'set version := "'$CONTINUATIONS_VER'"' $clean "${buildTasks[@]}" + 'set version := "'$CONTINUATIONS_VER'"' $clean test "${buildTasks[@]}" CONT_LIB_BUILT="yes" fi } @@ -266,7 +320,7 @@ buildSwing() { then echo "Found scala-swing $SWING_VER; not building." else update scala scala-swing "$SWING_REF" && gfxd - sbtBuild 'set version := "'$SWING_VER'"' $clean "${buildTasks[@]}" + sbtBuild 'set version := "'$SWING_VER'"' $clean test "${buildTasks[@]}" SWING_BUILT="yes" fi } @@ -276,6 +330,8 @@ buildActorsMigration(){ then echo "Found scala-actors-migration $ACTORS_MIGRATION_VER; not building." else update scala actors-migration "$ACTORS_MIGRATION_REF" && gfxd + # not running tests because + # [error] Test scala.actors.migration.NestedReact.testNestedReactAkka failed: java.util.concurrent.TimeoutException: Futures timed out after [20 seconds] sbtBuild 'set version := "'$ACTORS_MIGRATION_VER'"' 'set VersionKeys.continuationsVersion := "'$CONTINUATIONS_VER'"' $clean "${buildTasks[@]}" ACTORS_MIGRATION_BUILT="yes" fi @@ -307,7 +363,7 @@ buildModules() { ## BUILD STEPS: determineScalaVersion() { - update scala scala $SCALA_REF + cd $WORKSPACE parseScalaProperties "versions.properties" if [ -z "$SCALA_VER_BASE" ]; then @@ -327,7 +383,7 @@ determineScalaVersion() { # TODO: publish nightly snapshot using this script publishToSonatype="no" - echo "dist_ref=2.11.x" >> $baseDir/jenkins.properties # for the -dist downstream jobs that build the actual archives + echo "repo_ref=2.11.x" >> $baseDir/jenkins.properties # for the -dist downstream jobs that build the actual archives else echo "HEAD is tagged as $scalaTag." # borrowed from https://github.com/cloudflare/semver_bash/blob/master/semver.sh @@ -452,7 +508,7 @@ constructUpdatedModuleVersions() { bootstrap() { echo "### Bootstrapping" - update scala scala $SCALA_REF && gfxd + cd $WORKSPACE #### LOCKER @@ -480,7 +536,7 @@ bootstrap() { # we only need to build the modules necessary to build Scala itself # since the version of locker and quick are the same publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"private-repo\" at \"$privateRepo\")") - buildTasks=(test $publishPrivateTask) + buildTasks=($publishPrivateTask) buildModules constructUpdatedModuleVersions @@ -497,8 +553,9 @@ bootstrap() { # Sanity check: make sure the Scala test suite passes / docs can be generated with these modules. # don't skip locker (-Dlocker.skip=1), or stability will fail # overwrite "locker" version of scala at private-repo with bootstrapped version - cd $baseDir/scala - gfxd + cd $baseDir + rm -rf build/ # must leave everything else in $baseDir for downstream jobs + ant -Dstarr.version=$SCALA_VER\ -Dextra.repo.url=$privateRepo\ -Dmaven.version.suffix=$SCALA_VER_SUFFIX\ @@ -518,7 +575,7 @@ bootstrap() { # git commit versions.properties -m"Bump versions.properties for $SCALA_VER." } -# assumes we just bootstrapped, and current directory is $baseDir/scala +# assumes we just bootstrapped, and current directory is $baseDir # publishes locker to sonatype, then builds modules again (those for which version numbers were provided), # and publishes those to sonatype as well # finally, the staging repos are closed @@ -534,7 +591,7 @@ publishSonatype() { # NOTE: only publish those for which versions are set # test and publish to sonatype, assuming you have ~/.sbt/0.13/sonatype.sbt and ~/.sbt/0.13/plugin/gpg.sbt publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)") - buildTasks=(test $publishSonatypeTaskModules) + buildTasks=($publishSonatypeTaskModules) buildModules open=$(st_stagingReposOpen) @@ -561,7 +618,7 @@ if [ "$publishToSonatype" == "yes" ] then publishSonatype else # build modules one more time, just to mimic the regular build as much when running as nightly echo "### Rebuilding modules with quick, publishing to $baseDir/ivy/local" - buildTasks=(test publish-local) + buildTasks=(publish-local) # buildScalacheck always uses publishPrivateTask (not buildTasks). we override it to avoid publishing to private-repo. publishPrivateTask="publish-local" forceRebuild="yes" -- cgit v1.2.3