Towards travis-style CI

Move script to the repo it acts on, out of scala/jenkins-scripts

Had to make some changes:
 - scala checkout is now in root ($WORKSPACE)
 - credentials are actually in homedir (sonatype/private-repo)
 - inline pr-scala-common
 - don't run tests for actors-migration due to timeout (TODO: revert?)

2 files changed, 87 insertions, 290 deletions

diff --git a/scripts/jobs/scala-release-2.11.x-build b/scripts/jobs/scala-release-2.11.x-build
index 141ccb7930..21fbb8fa76 100755
--- a/scripts/jobs/scala-release-2.11.x-build
+++ b/scripts/jobs/scala-release-2.11.x-build
@@ -8,12 +8,12 @@
     #   <username>jenkinside</username>
     #   <password></password>
     # </server>
-# - ~/.ivy2/.credentials (for sonatype)
+# - ~/.credentials (for sonatype)
     # realm=Sonatype Nexus Repository Manager
     # host=oss.sonatype.org
     # user=lamp
     # password=
-# - ~/.ivy2/.credentials-private-repo for private-repo.typesafe.com, as follows:
+# - ~/.credentials-private-repo for private-repo.typesafe.com, as follows:
     # realm=Artifactory Realm
     # host=private-repo.typesafe.com
     # user=jenkinside
@@ -46,7 +46,7 @@
 # Rebuild modules with this Scala build, and stage them on sonatype as well
 # This script can be run in multiple modes. It is design to work without any input,
 # so that it could be run in Travis CI. In that mode, it'll build a release when
-# the current HEAD of $SCALA_REF is tagged, and stage to sonatype. Otherwise,
+# the current HEAD of the checkout in $WORKSPACE is tagged, and stage to sonatype. Otherwise,
 # it'll build a nightly.
 #
 # Since the nightlies are intended to be a drop in replacement, all modules are built with the
@@ -56,18 +56,17 @@
 # The other way to trigger a release is by setting the SCALA_VER_BASE env var.
 #
 # By default, we build the versions of the modules as specified by versions.properties
-# (as specified in the $SCALA_REF commit). Set moduleVersioning to something random
+# (as specified in the HEAD commit). Set moduleVersioning to something random
 # to trigger building HEAD of each module, generating a fresh -$(git describe)-nightly version for each.
 #
 # PS: set publishToSonatype to anything but "yes" to avoid publishing to sonatype
-# (publishing only done when $SCALA_REF's HEAD is tagged / SCALA_VER_BASE is set.)
+# (publishing only done when $WORKSPACE checkout's HEAD is tagged / SCALA_VER_BASE is set.)
 
-SCALA_REF=${SCALA_REF-"2.11.x"}
 
 # set to something besides the default to build nightly snapshots of the modules instead of some tagged version
 moduleVersioning=${moduleVersioning-"versions.properties"}
 
-baseDir=${baseDir-`pwd`}
+baseDir=${WORKSPACE-`pwd`}
 publishPrivateTask=${publishPrivateTask-"publish"}
 publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"}
 publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"}
@@ -86,12 +85,70 @@ forceRebuild=${forceRebuild-no}
 # overridden to "no" when no SCALA_VER_BASE is passed and HEAD is not tagged with a valid version tag
 #
 
-antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check
+antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak)
 clean="clean" # TESTING leave empty to speed up testing
 
-scriptsDir="$( cd "$( dirname "$0" )/.." && pwd )"
-. $scriptsDir/common
-. $scriptsDir/pr-scala-common
+scriptsDir="$WORKSPACE/scripts"
+
+# This is for forcibly stopping the job from a subshell (see test
+# below).
+trap "exit 1" TERM
+export TOP_PID=$$
+set -e
+
+# Known problems : does not fare well with interrupted, partial
+# compilations. We should perhaps have a multi-dependency version
+# of do_i_have below
+
+LOGGINGDIR="$WORKSPACE/logs"
+mkdir -p $LOGGINGDIR
+
+unset SBT_HOME
+SBT_HOME="$WORKSPACE/.sbt"
+mkdir -p $SBT_HOME
+IVY_CACHE="$WORKSPACE/.ivy2"
+mkdir -p $IVY_CACHE
+rm -rf $IVY_CACHE/cache/org.scala-lang
+
+# temp dir where all 'non-build' operation are performed
+TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX)
+TMP_DIR="${TMP_ROOT_DIR}/tmp"
+mkdir "${TMP_DIR}"
+
+
+# detect sed version and how to enable extended regexes
+SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)"
+
+
+
+# :docstring test:
+# Usage: test <argument ..>
+# Executes <argument ..>, logging the launch of the command to the
+# main log file, and kills global script execution with the TERM
+# signal if the commands ends up failing.
+# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES,
+# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST
+# :end docstring:
+
+function test() {
+    echo "### $@"
+    "$@"
+    status=$?
+    if [ $status -ne 0 ]; then
+        say "### ERROR with $1"
+        kill -s TERM $TOP_PID
+    fi
+}
+
+# :docstring say:
+# Usage: say <argument ..>
+# Prints <argument ..> to both console and the main log file.
+# :end docstring:
+
+function say(){
+    (echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log
+}
+
 
 # we must change ivy home to get a fresh ivy cache, otherwise we get half-bootstrapped scala
 # rm it in case it existed (and there's no ivy2-shadow, which indicates we're running in a TESTING environment)...
@@ -102,15 +159,12 @@ mkdir -p $baseDir/ivy2
 rm -rf $baseDir/resolutionScratch_
 mkdir -p $baseDir/resolutionScratch_
 
-mkdir -p $baseDir/logs
-
-
 # repo used to publish "locker" scala to (to start the bootstrap)
 privateCred="private-repo"
 privateRepo="http://private-repo.typesafe.com/typesafe/scala-release-temp/"
 
 function parseScalaProperties(){
-  propFile="$baseDir/scala/$1"
+  propFile="$baseDir/$1"
   if [ ! -f $propFile ]; then
     echo "Property file $propFile not found."
     exit 1
@@ -206,7 +260,7 @@ buildXML() {
   then echo "Found scala-xml $XML_VER; not building."
   else
     update scala scala-xml "$XML_REF" && gfxd
-    sbtBuild 'set version := "'$XML_VER'-DOC"' $clean doc  'set version := "'$XML_VER'"' "${buildTasks[@]}"
+    sbtBuild 'set version := "'$XML_VER'-DOC"' $clean doc  'set version := "'$XML_VER'"' test "${buildTasks[@]}"
     XML_BUILT="yes" # ensure the module is built and published when buildXML is invoked for the second time, see comment above
   fi
 }
@@ -216,7 +270,7 @@ buildParsers() {
   then echo "Found scala-parser-combinators $PARSERS_VER; not building."
   else
     update scala scala-parser-combinators "$PARSERS_REF" && gfxd
-    sbtBuild 'set version := "'$PARSERS_VER'-DOC"' $clean doc 'set version := "'$PARSERS_VER'"' "${buildTasks[@]}"
+    sbtBuild 'set version := "'$PARSERS_VER'-DOC"' $clean doc 'set version := "'$PARSERS_VER'"' test "${buildTasks[@]}"
     PARSERS_BUILT="yes"
   fi
 }
@@ -226,7 +280,7 @@ buildPartest() {
   then echo "Found scala-partest $PARTEST_VER; not building."
   else
     update scala scala-partest "$PARTEST_REF" && gfxd
-    sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' 'set VersionKeys.scalaCheckVersion := "'$SCALACHECK_VER'"' $clean "${buildTasks[@]}"
+    sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' 'set VersionKeys.scalaCheckVersion := "'$SCALACHECK_VER'"' $clean test "${buildTasks[@]}"
     PARTEST_BUILT="yes"
   fi
 }
@@ -247,7 +301,7 @@ buildContinuations() {
     update scala scala-continuations $CONTINUATIONS_REF && gfxd
 
     $sbtCmd $sbtArgs 'project plugin' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \
-      'set version := "'$CONTINUATIONS_VER'"' $clean "compile:package" "${buildTasks[@]}" # https://github.com/scala/scala-continuations/pull/4
+      'set version := "'$CONTINUATIONS_VER'"' $clean "compile:package" test "${buildTasks[@]}" # https://github.com/scala/scala-continuations/pull/4
     CONT_PLUG_BUILT="yes"
   fi
 
@@ -256,7 +310,7 @@ buildContinuations() {
   else
     update scala scala-continuations $CONTINUATIONS_REF && gfxd
     $sbtCmd $sbtArgs 'project library' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \
-      'set version := "'$CONTINUATIONS_VER'"' $clean "${buildTasks[@]}"
+      'set version := "'$CONTINUATIONS_VER'"' $clean test "${buildTasks[@]}"
     CONT_LIB_BUILT="yes"
   fi
 }
@@ -266,7 +320,7 @@ buildSwing() {
   then echo "Found scala-swing $SWING_VER; not building."
   else
     update scala scala-swing "$SWING_REF" && gfxd
-    sbtBuild 'set version := "'$SWING_VER'"' $clean "${buildTasks[@]}"
+    sbtBuild 'set version := "'$SWING_VER'"' $clean test "${buildTasks[@]}"
     SWING_BUILT="yes"
   fi
 }
@@ -276,6 +330,8 @@ buildActorsMigration(){
   then echo "Found scala-actors-migration $ACTORS_MIGRATION_VER; not building."
   else
     update scala actors-migration "$ACTORS_MIGRATION_REF" && gfxd
+    # not running tests because
+    # [error] Test scala.actors.migration.NestedReact.testNestedReactAkka failed: java.util.concurrent.TimeoutException: Futures timed out after [20 seconds]
     sbtBuild 'set version := "'$ACTORS_MIGRATION_VER'"' 'set VersionKeys.continuationsVersion := "'$CONTINUATIONS_VER'"' $clean "${buildTasks[@]}"
     ACTORS_MIGRATION_BUILT="yes"
   fi
@@ -307,7 +363,7 @@ buildModules() {
 ## BUILD STEPS:
 
 determineScalaVersion() {
-  update scala scala $SCALA_REF
+  cd $WORKSPACE
   parseScalaProperties "versions.properties"
 
   if [ -z "$SCALA_VER_BASE" ]; then
@@ -327,7 +383,7 @@ determineScalaVersion() {
 
       # TODO: publish nightly snapshot using this script
       publishToSonatype="no"
-      echo "dist_ref=2.11.x" >> $baseDir/jenkins.properties # for the -dist downstream jobs that build the actual archives
+      echo "repo_ref=2.11.x" >> $baseDir/jenkins.properties # for the -dist downstream jobs that build the actual archives
     else
       echo "HEAD is tagged as $scalaTag."
       # borrowed from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
@@ -452,7 +508,7 @@ constructUpdatedModuleVersions() {
 bootstrap() {
   echo "### Bootstrapping"
 
-  update scala scala $SCALA_REF && gfxd
+  cd $WORKSPACE
 
   #### LOCKER
 
@@ -480,7 +536,7 @@ bootstrap() {
   # we only need to build the modules necessary to build Scala itself
   # since the version of locker and quick are the same
   publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"private-repo\" at \"$privateRepo\")")
-  buildTasks=(test $publishPrivateTask)
+  buildTasks=($publishPrivateTask)
   buildModules
 
   constructUpdatedModuleVersions
@@ -497,8 +553,9 @@ bootstrap() {
   # Sanity check: make sure the Scala test suite passes / docs can be generated with these modules.
   # don't skip locker (-Dlocker.skip=1), or stability will fail
   # overwrite "locker" version of scala at private-repo with bootstrapped version
-  cd $baseDir/scala
-  gfxd
+  cd $baseDir
+  rm -rf build/ # must leave everything else in $baseDir for downstream jobs
+
   ant -Dstarr.version=$SCALA_VER\
       -Dextra.repo.url=$privateRepo\
       -Dmaven.version.suffix=$SCALA_VER_SUFFIX\
@@ -518,7 +575,7 @@ bootstrap() {
   # git commit versions.properties -m"Bump versions.properties for $SCALA_VER."
 }
 
-# assumes we just bootstrapped, and current directory is $baseDir/scala
+# assumes we just bootstrapped, and current directory is $baseDir
 # publishes locker to sonatype, then builds modules again (those for which version numbers were provided),
 # and publishes those to sonatype as well
 # finally, the staging repos are closed
@@ -534,7 +591,7 @@ publishSonatype() {
   # NOTE: only publish those for which versions are set
   # test and publish to sonatype, assuming you have ~/.sbt/0.13/sonatype.sbt and ~/.sbt/0.13/plugin/gpg.sbt
   publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)")
-  buildTasks=(test $publishSonatypeTaskModules)
+  buildTasks=($publishSonatypeTaskModules)
   buildModules
 
   open=$(st_stagingReposOpen)
@@ -561,7 +618,7 @@ if [ "$publishToSonatype" == "yes" ]
   then publishSonatype
   else # build modules one more time, just to mimic the regular build as much when running as nightly
     echo "### Rebuilding modules with quick, publishing to $baseDir/ivy/local"
-    buildTasks=(test publish-local)
+    buildTasks=(publish-local)
     # buildScalacheck always uses publishPrivateTask (not buildTasks). we override it to avoid publishing to private-repo.
     publishPrivateTask="publish-local"
     forceRebuild="yes"
diff --git a/scripts/pr-scala-common b/scripts/pr-scala-common
deleted file mode 100755
index 01112588f7..0000000000
--- a/scripts/pr-scala-common
+++ /dev/null
@@ -1,260 +0,0 @@
-# This is for forcibly stopping the job from a subshell (see test
-# below).
-trap "exit 1" TERM
-export TOP_PID=$$
-set -e
-
-# Known problems : does not fare well with interrupted, partial
-# compilations. We should perhaps have a multi-dependency version
-# of do_i_have below
-
-BASEDIR="$(pwd)"
-
-LOGGINGDIR="$BASEDIR/logs"
-mkdir -p $LOGGINGDIR
-
-unset SBT_HOME
-SBT_HOME="$BASEDIR/.sbt"
-mkdir -p $SBT_HOME
-IVY_CACHE="$BASEDIR/.ivy2"
-mkdir -p $IVY_CACHE
-rm -rf $IVY_CACHE/cache/org.scala-lang
-
-# temp dir where all 'non-build' operation are performed
-TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX)
-TMP_DIR="${TMP_ROOT_DIR}/tmp"
-mkdir "${TMP_DIR}"
-
-
-# detect sed version and how to enable extended regexes
-SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)"
-
-
-# :docstring getOrUpdate:
-# Usage : getOrUpdate <directory> <url> <reference> <n>
-#
-# Updates or clones the checkout of <reference> taken from the
-# git repo at <url> into the local directory
-# <directory> and cleans the checkout.
-#
-# :end docstring:
-
-function getOrUpdate(){
-    if [ ! -d $1 ]; then
-        git clone --depth 1 $2
-    fi
-    pushd $1
-
-    git fetch $2 $3
-
-    git checkout -q FETCH_HEAD
-
-    git reset --hard FETCH_HEAD
-
-    git clean -fxd
-
-    git log --oneline -1
-
-    git status
-
-    popd
-}
-
-
-# :docstring parse_properties:
-# Usage: parse_properties javaPropertyFile
-# Exports variables set in property file under $BASEDIR, mangling names by replacing dots with _
-# :end docstring:
-
-function parse_properties(){
-    propFile="$BASEDIR/$1"
-    if [ ! -f $propFile ]; then
-      say "Property file $propFile not found."
-      exit 1
-    else
-      awk -f "$scriptsDir/readproperties.awk" "$propFile" > "$propFile.sh"
-      . "$propFile.sh" # yeah yeah, not that secure, improvements welcome (I tried, but bash made me cry again)
-    fi
-}
-
-
-# :docstring test:
-# Usage: test <argument ..>
-# Executes <argument ..>, logging the launch of the command to the
-# main log file, and kills global script execution with the TERM
-# signal if the commands ends up failing.
-# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES,
-# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST
-# :end docstring:
-
-function test() {
-    echo "### $@"
-    "$@"
-    status=$?
-    if [ $status -ne 0 ]; then
-        say "### ERROR with $1"
-        kill -s TERM $TOP_PID
-    fi
-}
-
-# :docstring say:
-# Usage: say <argument ..>
-# Prints <argument ..> to both console and the main log file.
-# :end docstring:
-
-function say(){
-    (echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log
-}
-
-# General debug logging
-# $* - message
-function debug () {
-  echo "----- $*"
-}
-
-
-## TAKEN FROM UBER-BUILD, except that it "returns" (via $RES) true/false
-# Check if an artifact is available
-# $1 - groupId
-# $2 - artifacId
-# $3 - version
-# $4 - extra repository to look in (optional)
-# return value in $RES
-function checkAvailability () {
-  cd "${TMP_DIR}"
-  rm -rf *
-
-# pom file for the test project
-  cat > pom.xml << EOF
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <groupId>com.typesafe</groupId>
-  <artifactId>typesafeDummy</artifactId>
-  <packaging>war</packaging>
-  <version>1.0-SNAPSHOT</version>
-  <name>Dummy</name>
-  <url>http://127.0.0.1</url>
-  <dependencies>
-    <dependency>
-      <groupId>$1</groupId>
-      <artifactId>$2</artifactId>
-      <version>$3</version>
-    </dependency>
-  </dependencies>
-  <repositories>
-    <repository>
-      <id>sonatype.snapshot</id>
-      <name>Sonatype maven snapshot repository</name>
-      <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-      <snapshots>
-        <updatePolicy>daily</updatePolicy>
-      </snapshots>
-    </repository>
-EOF
-
-  if [ -n "$4" ]
-  then
-# adds the extra repository
-    cat >> pom.xml << EOF
-    <repository>
-      <id>extrarepo</id>
-      <name>extra repository</name>
-      <url>$4</url>
-    </repository>
-EOF
-  fi
-
-  cat >> pom.xml << EOF
-  </repositories>
-</project>
-EOF
-
-  set +e
-  mvn "${MAVEN_ARGS[@]}" compile &> "${TMP_DIR}/mvn.log"
-  RES=$?
-  # Quiet the maven, but allow diagnosing problems.
-  grep -i downloading "${TMP_DIR}/mvn.log"
-  grep -i exception "${TMP_DIR}/mvn.log"
-  grep -i error "${TMP_DIR}/mvn.log"
-  set -e
-
-# log the result
-  if [ ${RES} == 0 ]
-  then
-    debug "$1:$2:jar:$3 found !"
-    RES=true
-  else
-    debug "$1:$2:jar:$3 not found !"
-    RES=false
-  fi
-}
-
-
-# :docstring preparesbt:
-# Usage: preparesbt
-# This lets sbt know to look for the local maven repository.
-# :end docstring:
-
-# don't share any caches, sbt dirs, repos,... to avoid concurrent writes
-# keep them local to the workspace also lets us diagnose problems more easily
-# Make sure this is an absolute path with preceding '/'
-LOCAL_M2_REPO="$BASEDIR/m2repo"
-GENMVNOPTS="-e -B -X -Dmaven.repo.local=${LOCAL_M2_REPO}"
-# otherwise this just keeps growing and growing due to the -$sha-SNAPSHOT approach
-[[ -d "$LOCAL_M2_REPO"/org/scala-lang ]] && rm -rf "$LOCAL_M2_REPO"/org/scala-lang
-
-function preparesbt(){
-    # Am I using sbt-extras? I.e., do we need to use -sbt-dir?
-    set +e
-    sbt -h 2>&1 | grep -qe "-sbt-dir"
-    sbt_extraed=$?
-    set -e
-    export DEST_REPO_FILE=$SBT_HOME/repositories
-    if [ $sbt_extraed -eq 0 ]; then
-        # sbt-extras does not honor an explicit -Dsbt.global.base
-        export SBT_ARGS="-verbose -debug -no-colors -sbt-dir $SBT_HOME -ivy $IVY_CACHE"
-        say "### sbt-extras detected, using args $SBT_ARGS"
-    else
-        # don't pass -verbose or -debug, they tend to break sbt...
-        export SBT_ARGS="-Dsbt.global.base=$SBT_HOME -Dsbt.ivy.home=$IVY_CACHE"
-        say "### vanilla sbt detected, using args $SBT_ARGS"
-    fi
-
-    if [ -f $DEST_REPO_FILE ]; then
-        export OLD_SBT_REPO_FILE=$(mktemp -t sbtreposXXX)
-        cat $DEST_REPO_FILE > $OLD_SBT_REPO_FILE
-    fi
-    cat > $DEST_REPO_FILE <<EOF
-[repositories]
-  maven-central
-  local
-  typesafe-ivy-releases: http://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext]
-  sonatype-snapshots: https://oss.sonatype.org/content/repositories/snapshots
-  sonatype-releases: https://oss.sonatype.org/content/repositories/releases
-  mavenLocal: file://$LOCAL_M2_REPO
-  prRepo: $prRepoUrl
-EOF
-}
-
-# :docstring cleanupsbt:
-# Usage: cleanupsbt
-# This reestablishes the previous .sbt/repositories.
-# :end docstring:
-
-function cleanupsbt(){
-    say "### cleaning up $DEST_REPO_FILE"
-    if [[ ! -z $OLD_SBT_REPO_FILE ]]; then
-        mv $OLD_SBT_REPO_FILE $DEST_REPO_FILE
-    else
-        rm $DEST_REPO_FILE
-    fi
-}
-
-function main() {
-    SCALAHASH=$sha # passed in by jenkins
-    SCALADIR="$BASEDIR/scala/"
-    if [[ -z $SCALAHASH ]]; then
-        echo "No Scala sha provided!"
-        exit 1
-    fi
-}