From 706d68f863ebffb0f63811afe0835c5f975ba225 Mon Sep 17 00:00:00 2001 From: Adriaan Moors Date: Mon, 2 Feb 2015 22:59:15 -0800 Subject: New CI validation scripts Currently not validating the IDE, pending fix for https://github.com/scala-ide/uber-build/issues/48. The new infrastructure is documented over at: - https://github.com/scala/scabot - https://github.com/scala/scala-jenkins-infra - [jenkins jobs definitions](https://github.com/scala/scala-jenkins-infra/tree/master/templates/default/jobs/validate) --- scripts/common | 153 ++++++++++ scripts/jobs/integrate/bootstrap | 563 +++++++++++++++++++++++++++++++++++++ scripts/jobs/integrate/ide | 35 +++ scripts/jobs/validate/publish-core | 42 +++ scripts/jobs/validate/test | 17 ++ scripts/readproperties.awk | 39 +++ scripts/repositories-scala-release | 7 + 7 files changed, 856 insertions(+) create mode 100644 scripts/common create mode 100755 scripts/jobs/integrate/bootstrap create mode 100755 scripts/jobs/integrate/ide create mode 100755 scripts/jobs/validate/publish-core create mode 100755 scripts/jobs/validate/test create mode 100644 scripts/readproperties.awk create mode 100644 scripts/repositories-scala-release diff --git a/scripts/common b/scripts/common new file mode 100644 index 0000000000..b075469379 --- /dev/null +++ b/scripts/common @@ -0,0 +1,153 @@ +# This is for forcibly stopping the job from a subshell (see test +# below). +trap "exit 1" TERM +export TOP_PID=$$ +set -e + +# Known problems : does not fare well with interrupted, partial +# compilations. We should perhaps have a multi-dependency version +# of do_i_have below + +LOGGINGDIR="$WORKSPACE/logs" +mkdir -p $LOGGINGDIR + +unset SBT_HOME +SBT_HOME="$WORKSPACE/.sbt" +mkdir -p $SBT_HOME +IVY_CACHE="$WORKSPACE/.ivy2" +mkdir -p $IVY_CACHE +rm -rf $IVY_CACHE/cache/org.scala-lang + +# temp dir where all 'non-build' operation are performed +TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX) +TMP_DIR="${TMP_ROOT_DIR}/tmp" +mkdir "${TMP_DIR}" + + +# detect sed version and how to enable extended regexes +SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)" + + + +# :docstring test: +# Usage: test +# Executes , logging the launch of the command to the +# main log file, and kills global script execution with the TERM +# signal if the commands ends up failing. +# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES, +# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST +# :end docstring: + +function test() { + echo "### $@" + "$@" + status=$? + if [ $status -ne 0 ]; then + say "### ERROR with $1" + kill -s TERM $TOP_PID + fi +} + +# :docstring say: +# Usage: say +# Prints to both console and the main log file. +# :end docstring: + +function say(){ + (echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log +} + +# General debug logging +# $* - message +function debug () { + echo "----- $*" +} + +function parseScalaProperties(){ + propFile="$baseDir/$1" + if [ ! -f $propFile ]; then + echo "Property file $propFile not found." + exit 1 + else + awk -f "$scriptsDir/readproperties.awk" "$propFile" > "$propFile.sh" + . "$propFile.sh" # yeah yeah, not that secure, improvements welcome (I tried, but bash made me cry again) + fi +} + + +## TAKEN FROM UBER-BUILD, except that it "returns" (via $RES) true/false +# Check if an artifact is available +# $1 - groupId +# $2 - artifacId +# $3 - version +# $4 - extra repository to look in (optional) +# return value in $RES +function checkAvailability () { + pushd "${TMP_DIR}" + rm -rf * + +# pom file for the test project + cat > pom.xml << EOF + + 4.0.0 + com.typesafe + typesafeDummy + war + 1.0-SNAPSHOT + Dummy + http://127.0.0.1 + + + $1 + $2 + $3 + + + + + sonatype.snapshot + Sonatype maven snapshot repository + https://oss.sonatype.org/content/repositories/snapshots + + daily + + +EOF + + if [ -n "$4" ] + then +# adds the extra repository + cat >> pom.xml << EOF + + extrarepo + extra repository + $4 + +EOF + fi + + cat >> pom.xml << EOF + + +EOF + + set +e + mvn "${MAVEN_ARGS[@]}" compile &> "${TMP_DIR}/mvn.log" + RES=$? + # Quiet the maven, but allow diagnosing problems. + grep -i downloading "${TMP_DIR}/mvn.log" + grep -i exception "${TMP_DIR}/mvn.log" + grep -i error "${TMP_DIR}/mvn.log" + set -e + +# log the result + if [ ${RES} == 0 ] + then + debug "$1:$2:jar:$3 found !" + RES=true + else + debug "$1:$2:jar:$3 not found !" + RES=false + fi + popd +} diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap new file mode 100755 index 0000000000..46d610018c --- /dev/null +++ b/scripts/jobs/integrate/bootstrap @@ -0,0 +1,563 @@ +#!/bin/bash -e +# TODO: different scripts for the different phases -- usually we don't need to bootstrap the modules, +# since we can use the previous version of scala for STARR as well as for compiling the modules (assuming it's binary compatible) +# We should move away from the complicated bootstrap and set up our release schedule so we always have a previous build that satisfies these criteria. +# (Potentially trivially, by splitting up this script, and publishing locker as if it were a real release.) + +# requirements: +# sbtCmd must point to sbt from sbt-extras (this is the standard on the Scala jenkins, so we only support that one) +# - ~/.sonatype-curl that consists of user = USER:PASS +# - ~/.m2/settings.xml with credentials for sonatype + # + # private-repo + # jenkinside + # + # +# - ~/.credentials (for sonatype) + # realm=Sonatype Nexus Repository Manager + # host=oss.sonatype.org + # user=lamp + # password= +# - ~/.credentials-private-repo for private-repo.typesafe.com, as follows: + # realm=Artifactory Realm + # host=private-repo.typesafe.com + # user=jenkinside + # password= +# - ~/.sbt/0.13/plugins/gpg.sbt with: +# addSbtPlugin("com.typesafe.sbt" % "sbt-pgp" % "0.8.1") + +# Modus operandi: +# +# Determine Scala version as: +# +# $SCALA_VER_BASE$SCALA_VER_SUFFIX (if former variable is set) +# By parsing the tag (if HEAD is tagged as v$base$suffix) +# By parsing build.number for the base version, suffixing with -$sha-nightly +# Serialize these versions to jenkins.properties, which are passed downstream to scala-release-2.11.x-dist. +# This also removes the need to tag scala/scala-dist (not possible for nightlies, still encouraged for releases, but not a hard requirement). +# +# Determine Module Versions +# +# When running in "versions.properties" mode (the default), derive tags from these versions and build, publishing only those modules that are not available yet. +# Otherwise, build HEAD for all modules, derive a -nightly version for them. +# Bootstrap: +# +# Build minimal core of Scala as this version (aka locker), publish to private-repo +# Build modules required to bootstrap, publish to private-repo +# Build Scala using the previously built core and bootstrap modules, publish to private-repo This overwrites the minimal core on private-repo +# Stage to sonatype (unless building a -nightly release): +# +# Stage this Scala build on sonatype +# Rebuild modules with this Scala build, and stage them on sonatype as well +# This script can be run in multiple modes. It is design to work without any input, +# so that it could be run in Travis CI. In that mode, it'll build a release when +# the current HEAD of the checkout in $WORKSPACE is tagged, and stage to sonatype. Otherwise, +# it'll build a nightly. +# +# Since the nightlies are intended to be a drop in replacement, all modules are built with the +# full Scala version as their binary version, so that you can just set scalaVersion to the +# nightly's sha-derived version and be good to go. +# +# The other way to trigger a release is by setting the SCALA_VER_BASE env var. +# +# By default, we build the versions of the modules as specified by versions.properties +# (as specified in the HEAD commit). Set moduleVersioning to something random +# to trigger building HEAD of each module, generating a fresh -$(git describe)-nightly version for each. +# +# PS: set publishToSonatype to anything but "yes" to avoid publishing to sonatype +# (publishing only done when $WORKSPACE checkout's HEAD is tagged / SCALA_VER_BASE is set.) + + +# set to something besides the default to build nightly snapshots of the modules instead of some tagged version +moduleVersioning=${moduleVersioning-"versions.properties"} + +publishPrivateTask=${publishPrivateTask-"publish"} +publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"} +publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"} +publishLockerPrivateTask=${publishLockerPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built locker before) + +sbtCmd=${sbtCmd-sbt} # TESTING (this is a marker for defaults to change when testing locally: should be sbtx on my mac) + +# 0.13.5 does not respect "set every scalaVersion", see +# https://github.com/scala/scala-parser-combinators/pull/27 +sbtCmd="$sbtCmd -sbt-version 0.13.2" + +forceRebuild=${forceRebuild-no} + +# publishToSonatype +# set to anything but "yes" to avoid publishing to sonatype +# overridden to "no" when no SCALA_VER_BASE is passed and HEAD is not tagged with a valid version tag +# + +antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak) +clean="clean" # TESTING leave empty to speed up testing + + + +baseDir=${WORKSPACE-`pwd`} +scriptsDir="$baseDir/scripts" +. $scriptsDir/common + +# we must change ivy home to get a fresh ivy cache, otherwise we get half-bootstrapped scala +# rm it in case it existed (and there's no ivy2-shadow, which indicates we're running in a TESTING environment)... +# we don't nuke the whole ws since that clobbers the git clones needlessly +[[ -d $baseDir/ivy2-shadow ]] || rm -rf $baseDir/ivy2 +mkdir -p $baseDir/ivy2 + +rm -rf $baseDir/resolutionScratch_ +mkdir -p $baseDir/resolutionScratch_ + +# repo used to publish "locker" scala to (to start the bootstrap) +privateCred="private-repo" +privateRepo="http://private-repo.typesafe.com/typesafe/scala-release-temp/" + +##### git +gfxd() { + git clean -fxd # TESTING +} + +update() { + [[ -d $baseDir ]] || mkdir -p $baseDir + cd $baseDir + + if [ ! -d $baseDir/$2 ]; then git clone "https://github.com/$1/$2.git"; fi + + cd $2 + + git fetch --tags "https://github.com/$1/$2.git" + (git fetch "https://github.com/$1/$2.git" $3 && git checkout -q FETCH_HEAD) #|| git checkout -q $3 # || fallback is for local testing on tag + git reset --hard +} + +##### sonatype interface + +stApi="https://oss.sonatype.org/service/local" + +function st_curl(){ + curl -H "Content-Type: application/json" -H "accept: application/json,application/vnd.siesta-error-v1+json,application/vnd.siesta-validation-errors-v1+json" -K ~/.sonatype-curl -s -o - $@ +} + +function st_stagingReposOpen() { + st_curl "$stApi/staging/profile_repositories" | jq '.data[] | select(.profileName == "org.scala-lang") | select(.type == "open")' +} + +function st_stagingRepoDrop() { + repo=$1 + message=$2 + echo "{\"data\":{\"description\":\"$message\",\"stagedRepositoryIds\":[\"$repo\"]}}" | st_curl -X POST -d @- "$stApi/staging/bulk/drop" +} + +function st_stagingRepoClose() { + repo=$1 + message=$2 + echo "{\"data\":{\"description\":\"$message\",\"stagedRepositoryIds\":[\"$repo\"]}}" | st_curl -X POST -d @- "$stApi/staging/bulk/close" +} + + +# ARGH trying to get this to work on multiple versions of sbt-extras... +# the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir +# the new version of sbt-extras ignores sbt-dir, so we pass it in as -Dsbt.global.base +# need to set sbt-dir to one that has the gpg.sbt plugin config +sbtArgs="-no-colors -ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$scriptsDir/repositories-scala-release -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13" + +sbtBuild() { + echo "### sbtBuild: "$sbtCmd $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" + $sbtCmd $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1 +} + +sbtResolve() { + cd $baseDir/resolutionScratch_ + touch build.sbt + cross=${4-binary} # Disabled / binary / full + echo "### sbtResolve: $sbtCmd $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" + $sbtCmd $sbtArgs "${scalaVersionTasks[@]}" \ + "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \ + 'show update' >> $baseDir/logs/resolution 2>&1 +} + +# Oh boy... can't use scaladoc to document scala-xml/scala-parser-combinators +# if scaladoc depends on the same version of scala-xml/scala-parser-combinators. +# Even if that version is available through the project's resolvers, sbt won't look past this project. +# SOOOOO, we set the version to a dummy (-DOC), generate documentation, +# then set the version to the right one and publish (which won't re-gen the docs). +# Also tried publish-local without docs using 'set publishArtifact in (Compile, packageDoc) := false' and republishing, no dice. + +# Each buildModule() function is invoked twice: first to build against locker and publish to private-repo, then +# to build against the release and publish to sonatype (or publish-local if publishToSonatype is not "yes"). +# In the second round, sbtResolve is always true: the module will be found in the private-repo! +# Therefore, if MODULE_BUILT is "yes" (in the second round), we know that we need to build (and publish) the +# module again. +# +# Note: we tried an alternative solution in which sbtResolve would not look at private-repo, but that fails. For example, +# scala-xml depends on scala-library, so sbt tries to find the scala-library of the version that we are currently building, +# which exists only in private-repo. + +buildXML() { + if [ "$XML_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-xml" $XML_VER ) + then echo "Found scala-xml $XML_VER; not building." + else + update scala scala-xml "$XML_REF" && gfxd + sbtBuild 'set version := "'$XML_VER'-DOC"' $clean doc 'set version := "'$XML_VER'"' test "${buildTasks[@]}" + XML_BUILT="yes" # ensure the module is built and published when buildXML is invoked for the second time, see comment above + fi +} + +buildParsers() { + if [ "$PARSERS_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-parser-combinators" $PARSERS_VER ) + then echo "Found scala-parser-combinators $PARSERS_VER; not building." + else + update scala scala-parser-combinators "$PARSERS_REF" && gfxd + sbtBuild 'set version := "'$PARSERS_VER'-DOC"' $clean doc 'set version := "'$PARSERS_VER'"' test "${buildTasks[@]}" + PARSERS_BUILT="yes" + fi +} + +buildPartest() { + if [ "$PARTEST_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-partest" $PARTEST_VER ) + then echo "Found scala-partest $PARTEST_VER; not building." + else + update scala scala-partest "$PARTEST_REF" && gfxd + sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' 'set VersionKeys.scalaCheckVersion := "'$SCALACHECK_VER'"' $clean test "${buildTasks[@]}" + PARTEST_BUILT="yes" + fi +} + +# buildPartestIface() { +# if [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-partest-interface" $PARTEST_IFACE_VER ) +# then echo "Found scala-partest-interface $PARTEST_IFACE_VER; not building." +# else +# update scala scala-partest-interface "$PARTEST_IFACE_REF" && gfxd +# sbtBuild 'set version :="'$PARTEST_IFACE_VER'"' $clean "${buildTasks[@]}" +# fi +# } + +buildContinuations() { + if [ "$CONT_PLUG_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.plugins" "scala-continuations-plugin" $CONTINUATIONS_VER full ) + then echo "Found scala-continuations-plugin $CONTINUATIONS_VER; not building." + else + update scala scala-continuations $CONTINUATIONS_REF && gfxd + + $sbtCmd $sbtArgs 'project plugin' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ + 'set version := "'$CONTINUATIONS_VER'"' $clean "compile:package" test "${buildTasks[@]}" # https://github.com/scala/scala-continuations/pull/4 + CONT_PLUG_BUILT="yes" + fi + + if [ "$CONT_LIB_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.plugins" "scala-continuations-library" $CONTINUATIONS_VER ) + then echo "Found scala-continuations-library $CONTINUATIONS_VER; not building." + else + update scala scala-continuations $CONTINUATIONS_REF && gfxd + $sbtCmd $sbtArgs 'project library' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ + 'set version := "'$CONTINUATIONS_VER'"' $clean test "${buildTasks[@]}" + CONT_LIB_BUILT="yes" + fi +} + +buildSwing() { + if [ "$SWING_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-swing" $SWING_VER ) + then echo "Found scala-swing $SWING_VER; not building." + else + update scala scala-swing "$SWING_REF" && gfxd + sbtBuild 'set version := "'$SWING_VER'"' $clean test "${buildTasks[@]}" + SWING_BUILT="yes" + fi +} + +buildActorsMigration(){ + if [ "$ACTORS_MIGRATION_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang" "scala-actors-migration" $ACTORS_MIGRATION_VER ) + then echo "Found scala-actors-migration $ACTORS_MIGRATION_VER; not building." + else + update scala actors-migration "$ACTORS_MIGRATION_REF" && gfxd + # not running tests because + # [error] Test scala.actors.migration.NestedReact.testNestedReactAkka failed: java.util.concurrent.TimeoutException: Futures timed out after [20 seconds] + sbtBuild 'set version := "'$ACTORS_MIGRATION_VER'"' 'set VersionKeys.continuationsVersion := "'$CONTINUATIONS_VER'"' $clean "${buildTasks[@]}" + ACTORS_MIGRATION_BUILT="yes" + fi +} + +buildScalacheck(){ + if [ "$SCALACHECK_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scalacheck" "scalacheck" $SCALACHECK_VER ) + then echo "Found scalacheck $SCALACHECK_VER; not building." + else + update rickynils scalacheck $SCALACHECK_REF && gfxd + sbtBuild 'set version := "'$SCALACHECK_VER'"' 'set VersionKeys.scalaParserCombinatorsVersion := "'$PARSERS_VER'"' $clean $publishPrivateTask # test times out NOTE: never published to sonatype + SCALACHECK_BUILT="yes" + fi +} + +# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to private-repo) +buildModules() { + buildXML + buildParsers + buildContinuations + buildSwing + buildActorsMigration + buildScalacheck + buildPartest + # buildPartestIface +} + + +## BUILD STEPS: + +determineScalaVersion() { + cd $WORKSPACE + parseScalaProperties "versions.properties" + + if [ -z "$SCALA_VER_BASE" ]; then + echo "No SCALA_VER_BASE specified." + + scalaTag=$(git describe --exact-match ||:) + + SCALA_BINARY_VER=${SCALA_BINARY_VER-"$scala_binary_version"} + + if [ -z "$scalaTag" ] + then + echo "No tag found, building nightly snapshot." + parseScalaProperties "build.number" + SCALA_VER_BASE="$version_major.$version_minor.$version_patch" + SCALA_VER_SUFFIX="-$(git rev-parse --short HEAD)-nightly" + SCALADOC_SOURCE_LINKS_VER=$(git rev-parse HEAD) + + # TODO: publish nightly snapshot using this script + publishToSonatype="no" + echo "repo_ref=2.11.x" >> $baseDir/jenkins.properties # for the -dist downstream jobs that build the actual archives + else + echo "HEAD is tagged as $scalaTag." + # borrowed from https://github.com/cloudflare/semver_bash/blob/master/semver.sh + local RE='v*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' # don't change this to make it more accurate, it's not worth it + SCALA_VER_BASE="$(echo $scalaTag | sed -e "s#$RE#\1.\2.\3#")" + SCALA_VER_SUFFIX="$(echo $scalaTag | sed -e "s#$RE#\4#")" + SCALADOC_SOURCE_LINKS_VER=$scalaTag + + if [ "$SCALA_VER_BASE" == "$scalaTag" ]; then + echo "Could not parse version $scalaTag" + exit 1 + fi + publishToSonatype=${publishToSonatype-"yes"} # unless forced previously, publish + fi + else + publishToSonatype=${publishToSonatype-"yes"} # unless forced previously, publish + # if version base/suffix are provided, we assume a corresponding tag exists for the scaladoc source links + SCALADOC_SOURCE_LINKS_VER="v$SCALA_VER_BASE$SCALA_VER_SUFFIX" + fi + + SCALA_VER="$SCALA_VER_BASE$SCALA_VER_SUFFIX" + echo "version=$SCALA_VER" >> $baseDir/jenkins.properties + echo "sbtDistVersionOverride=-Dproject.version=$SCALA_VER" >> $baseDir/jenkins.properties + + # We don't override the scala binary version: when running in -nightly + versions.properties versioning mode, + # we intend to be a drop-in replacement -- all you need to do is change the Scala version + # In order to override this, add 'set every scalaBinaryVersion := "'$SCALA_BINARY_VER'"', + # which, when used with pre-release Scala version numbers, will require tweaking at the sbt usage site as well. + scalaVersionTasks=('set every scalaVersion := "'$SCALA_VER'"') + + echo "Building Scala $SCALA_VER." +} + +deriveVersion() { + update $1 $2 $3 &> /dev/null + echo "$(git describe --match=v* | cut -dv -f2)-nightly" +} + +deriveVersionAnyTag() { + update $1 $2 $3 &> /dev/null + echo "$(git describe | cut -dv -f2)-nightly" +} + +# determineScalaVersion must have been called +deriveModuleVersions() { + if [ "$moduleVersioning" == "versions.properties" ] + then + # use versions.properties as defaults when no version specified on command line + XML_VER=${XML_VER-$scala_xml_version_number} + PARSERS_VER=${PARSERS_VER-$scala_parser_combinators_version_number} + CONTINUATIONS_VER=${CONTINUATIONS_VER-$scala_continuations_plugin_version_number} + SWING_VER=${SWING_VER-$scala_swing_version_number} + ACTORS_MIGRATION_VER=${ACTORS_MIGRATION_VER-$actors_migration_version_number} + PARTEST_VER=${PARTEST_VER-$partest_version_number} + SCALACHECK_VER=${SCALACHECK_VER-$scalacheck_version_number} + + # If a _VER was not specified, the corresponding _REF will be non-empty by now (as specified, or HEAD) + XML_REF=${XML_REF-"v$XML_VER"} + PARSERS_REF=${PARSERS_REF-"v$PARSERS_VER"} + CONTINUATIONS_REF=${CONTINUATIONS_REF-"v$CONTINUATIONS_VER"} + SWING_REF=${SWING_REF-"v$SWING_VER"} + ACTORS_MIGRATION_REF=${ACTORS_MIGRATION_REF-"v$ACTORS_MIGRATION_VER"} + PARTEST_REF=${PARTEST_REF-"v$PARTEST_VER"} + # PARTEST_IFACE_REF=${PARTEST_IFACE_REF-"v$PARTEST_IFACE_VER"} + SCALACHECK_REF=${SCALACHECK_REF-"$SCALACHECK_VER"} + else + XML_VER=${XML_VER-$(deriveVersion scala scala-xml "$XML_REF")} + PARSERS_VER=${PARSERS_VER-$(deriveVersion scala scala-parser-combinators "$PARSERS_REF")} + CONTINUATIONS_VER=${CONTINUATIONS_VER-$(deriveVersion scala scala-continuations "$CONTINUATIONS_REF")} + SWING_VER=${SWING_VER-$(deriveVersion scala scala-swing "$SWING_REF")} + ACTORS_MIGRATION_VER=${ACTORS_MIGRATION_VER-$(deriveVersion scala actors-migration "$ACTORS_MIGRATION_REF")} + PARTEST_VER=${PARTEST_VER-$(deriveVersion scala scala-partest "$PARTEST_REF")} + SCALACHECK_VER=${SCALACHECK_VER-$(deriveVersionAnyTag rickynils scalacheck "$SCALACHECK_REF")} + + XML_REF=${XML_REF-"HEAD"} + PARSERS_REF=${PARSERS_REF-"HEAD"} + CONTINUATIONS_REF=${CONTINUATIONS_REF-"HEAD"} + SWING_REF=${SWING_REF-"HEAD"} + ACTORS_MIGRATION_REF=${ACTORS_MIGRATION_REF-"HEAD"} + PARTEST_REF=${PARTEST_REF-"HEAD"} + # PARTEST_IFACE_REF=${PARTEST_IFACE_REF-"HEAD"} + SCALACHECK_REF=${SCALACHECK_REF-"HEAD"} + fi + + echo "Module versions (versioning strategy: $moduleVersioning):" + echo "ACTORS_MIGRATION = $ACTORS_MIGRATION_VER at $ACTORS_MIGRATION_REF" + echo "CONTINUATIONS = $CONTINUATIONS_VER at $CONTINUATIONS_REF" + echo "PARSERS = $PARSERS_VER at $PARSERS_REF" + echo "PARTEST = $PARTEST_VER at $PARTEST_REF" + echo "SCALACHECK = $SCALACHECK_VER at $SCALACHECK_REF" + echo "SWING = $SWING_VER at $SWING_REF" + echo "XML = $XML_VER at $XML_REF" + + # PARTEST_IFACE_VER=${PARTEST_IFACE_VER-$(deriveVersion scala scala-partest-interface "$PARTEST_IFACE_REF")} +} + +constructUpdatedModuleVersions() { + updatedModuleVersions=() + + # force the new module versions for building the core. these may be different from the values in versions.properties, + # either because the variables (XML_VER) were provided, or because we're building the modules from HEAD. + # in the common case, the values are the same as in versions.properties. + updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dactors-migration.version.number=$ACTORS_MIGRATION_VER") + updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-continuations-library.version.number=$CONTINUATIONS_VER") + updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-continuations-plugin.version.number=$CONTINUATIONS_VER") + updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-parser-combinators.version.number=$PARSERS_VER") + updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-swing.version.number=$SWING_VER") + updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-xml.version.number=$XML_VER") + + updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dpartest.version.number=$PARTEST_VER") + updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscalacheck.version.number=$SCALACHECK_VER") + + # allow overriding the akka-actors and jline version using a jenkins build parameter + if [ ! -z "$AKKA_ACTOR_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dakka-actor.version.number=$AKKA_ACTOR_VER"); fi + if [ ! -z "$JLINE_VER" ] ; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Djline.version=$JLINE_VER"); fi + + if [ ! -z "$SCALA_BINARY_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala.binary.version=$SCALA_BINARY_VER"); fi + if [ ! -z "$SCALA_FULL_VER" ] ; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala.full.version=$SCALA_FULL_VER"); fi +} + +# build locker (scala + modules) and quick, publishing everything to private-repo +bootstrap() { + echo "### Bootstrapping" + + cd $WORKSPACE + + #### LOCKER + + echo "### Building locker" + + # for bootstrapping, publish core (or at least smallest subset we can get away with) + # so that we can build modules with this version of Scala and publish them locally + # must publish under $SCALA_VER so that the modules will depend on this (binary) version of Scala + # publish more than just core: partest needs scalap + # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler + ant -Dmaven.version.number=$SCALA_VER\ + -Dremote.snapshot.repository=NOPE\ + -Dremote.release.repository=$privateRepo\ + -Drepository.credentials.id=$privateCred\ + -Dscalac.args.optimise=-optimise\ + -Ddocs.skip=1\ + -Dlocker.skip=1\ + $publishLockerPrivateTask >> $baseDir/logs/builds 2>&1 + + + echo "### Building modules using locker" + + # build, test and publish modules with this core + # publish to our internal repo (so we can resolve the modules in the scala build below) + # we only need to build the modules necessary to build Scala itself + # since the version of locker and quick are the same + publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"private-repo\" at \"$privateRepo\")") + buildTasks=($publishPrivateTask) + buildModules + + constructUpdatedModuleVersions + + #### QUICK + + echo "### Bootstrapping Scala using locker" + + # # TODO: close all open staging repos so that we can be reaonably sure the only open one we see after publishing below is ours + # # the ant call will create a new one + # + # Rebuild Scala with these modules so that all binary versions are consistent. + # Update versions.properties to new modules. + # Sanity check: make sure the Scala test suite passes / docs can be generated with these modules. + # don't skip locker (-Dlocker.skip=1), or stability will fail + # overwrite "locker" version of scala at private-repo with bootstrapped version + cd $baseDir + rm -rf build/ # must leave everything else in $baseDir for downstream jobs + + ant -Dstarr.version=$SCALA_VER\ + -Dextra.repo.url=$privateRepo\ + -Dmaven.version.suffix=$SCALA_VER_SUFFIX\ + ${updatedModuleVersions[@]} \ + -Dupdate.versions=1\ + -Dscaladoc.git.commit=$SCALADOC_SOURCE_LINKS_VER\ + -Dremote.snapshot.repository=NOPE\ + -Dremote.release.repository=$privateRepo\ + -Drepository.credentials.id=$privateCred\ + -Dscalac.args.optimise=-optimise\ + $antBuildTask $publishPrivateTask + + # clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala + rm -rf $baseDir/ivy2 + + # TODO: create PR with following commit (note that release will have been tagged already) + # git commit versions.properties -m"Bump versions.properties for $SCALA_VER." +} + +# assumes we just bootstrapped, and current directory is $baseDir +# publishes locker to sonatype, then builds modules again (those for which version numbers were provided), +# and publishes those to sonatype as well +# finally, the staging repos are closed +publishSonatype() { + # stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary, + # since we're just publishing an existing build + echo "### Publishing core to sonatype" + ant -Dmaven.version.number=$SCALA_VER $publishSonatypeTaskCore + + echo "### Publishing modules to sonatype" + # build/test/publish scala core modules to sonatype (this will start a new staging repo) + # (was hoping we could make everything go to the same staging repo, but it's not timing that causes two staging repos to be opened) + # NOTE: only publish those for which versions are set + # test and publish to sonatype, assuming you have ~/.sbt/0.13/sonatype.sbt and ~/.sbt/0.13/plugin/gpg.sbt + publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)") + buildTasks=($publishSonatypeTaskModules) + buildModules + + open=$(st_stagingReposOpen) + allOpenUrls=$(echo $open | jq '.repositoryURI' | tr -d \") + allOpen=$(echo $open | jq '.repositoryId' | tr -d \") + + echo "Closing open repos: $allOpen" + + for repo in $allOpen; do st_stagingRepoClose $repo; done + + echo "Closed sonatype staging repos: $allOpenUrls." +} + + +#### MAIN + +determineScalaVersion + +deriveModuleVersions + +bootstrap + +if [ "$publishToSonatype" == "yes" ] + then publishSonatype + else # build modules one more time, just to mimic the regular build as much when running as nightly + echo "### Rebuilding modules with quick, publishing to $baseDir/ivy/local" + buildTasks=(publish-local) + # buildScalacheck always uses publishPrivateTask (not buildTasks). we override it to avoid publishing to private-repo. + publishPrivateTask="publish-local" + forceRebuild="yes" + buildModules +fi diff --git a/scripts/jobs/integrate/ide b/scripts/jobs/integrate/ide new file mode 100755 index 0000000000..4ead284b9c --- /dev/null +++ b/scripts/jobs/integrate/ide @@ -0,0 +1,35 @@ +#!/bin/bash -e +# requires checkout: root is a scala checkout with which to integrate (actually, only required file is versions.properties, as documented below) +# requires env: scalaVersion (specifies binary already built from above checkout), WORKSPACE (provided by jenkins), repo_ref (HEAD of the scala checkout), +# requires files: $baseDir/versions.properties (from checkout -- defines version numbers for modules used to build scala for dbuild...) + +echo "IDE integration currently broken on 2.10.x. See https://github.com/scala-ide/uber-build/issues/48. Punting." +exit 0 + +# TODO: remove when integration is up and running +if [ "woele$_scabot_last" != "woele1" ]; then echo "Scabot didn't mark this as last commit -- skipping."; exit 0; fi + +baseDir=${WORKSPACE-`pwd`} +uberBuildUrl=${uberBuildUrl-"https://github.com/scala-ide/uber-build.git"} +uberBuildConfig=${uberBuildConfig-"validator-2.10.conf"} + +uberBuildDir="$baseDir/uber-build/" + +cd $baseDir +if [[ -d $uberBuildDir ]]; then + ( cd $uberBuildDir && git fetch $uberBuildUrl HEAD && git checkout -f FETCH_HEAD && git clean -fxd ) +else + git clone $uberBuildUrl +fi + +echo "maven.version.number=$scalaVersion" >> versions.properties + +# pass prRepoUrl in, which uber-build passes along to dbuild (in sbt-builds-for-ide) +# the "-P pr-scala" maven arg accomplishes the same thing for maven (directly used in uber-build) +BASEDIR="$baseDir" prRepoUrl="$prRepoUrl" MAVEN_ARGS="-P pr-scala"\ + $uberBuildDir/uber-build.sh $uberBuildDir/config/$uberBuildConfig $repo_ref $scalaVersion + +# uber-build puts its local repo under target/m2repo +# wipe the org/scala-lang part, which otherwise just keeps +# growing and growing due to the -$sha-SNAPSHOT approach +[[ -d $baseDir/target/m2repo/org/scala-lang ]] && rm -rf $baseDir/target/m2repo/org/scala-lang diff --git a/scripts/jobs/validate/publish-core b/scripts/jobs/validate/publish-core new file mode 100755 index 0000000000..392d01a591 --- /dev/null +++ b/scripts/jobs/validate/publish-core @@ -0,0 +1,42 @@ +#!/bin/bash -e +# This script publishes the core of Scala to maven for use as locker downstream, +# and saves the relevant properties used in its build artifacts, versions.properties. +# (This means we'll use locker instead of quick downstream in dbuild. +# The only downside is that backend improvements don't improve compiler performance itself until they are in STARR). +# The version is suffixed with "-${sha:0:7}-SNAPSHOT" + +baseDir=${WORKSPACE-`pwd`} +scriptsDir="$baseDir/scripts" +. $scriptsDir/common + +case $prDryRun in + yep) + echo "DRY RUN" + mkdir -p build/pack ; mkdir -p dists/maven/latest + ;; + *) + sha=$(git rev-parse HEAD) # TODO: warn if $repo_ref != $sha (we shouldn't do PR validation using symbolic gitrefs) + echo "sha/repo_ref == $sha/$repo_ref ?" + + parseScalaProperties build.number + + ./pull-binary-libs.sh + # "noyoudont" is there juuuust in case + antDeployArgs="-Dmaven.version.suffix=\"-${sha:0:7}-SNAPSHOT\" -Dremote.snapshot.repository=$prRepoUrl -Drepository.credentials.id=pr-scala -Dremote.release.repository=noyoudont" + + # master doesn't currently build with starr, so can't skip locker yet... + # TODO: enable for 2.11 once we have bumped versions in versions.properties accordingly + antBuildArgs="-Dlocker.skip=1" # -Dstarr.use.released=1 should be left up to the build (via versions.properties) + + # TODO: can we move doc generation downstream as well? + # build to (later) publish with a maven suffix that encodes the first 7 characters of the sha of the commit that we're validating + # (don't use the sha of the merge commit as we don't have an easy way of passing that down the validation chain) + ant -Darchives.skipxz=true $antDeployArgs $antBuildArgs distpack-maven-opt + + # mv buildcharacter.properties jenkins.properties # parsed by the jenkins job + echo "maven.version.number=$version_major.$version_minor.$version_patch-${sha:0:7}-SNAPSHOT" > jenkins.properties + + cd dists/maven/latest + ant $antDeployArgs deploy-core.snapshot + ;; +esac diff --git a/scripts/jobs/validate/test b/scripts/jobs/validate/test new file mode 100755 index 0000000000..c1c02c80cb --- /dev/null +++ b/scripts/jobs/validate/test @@ -0,0 +1,17 @@ +#!/bin/bash -e + +case $prDryRun in + yep) + echo "DRY RUN" + ;; + *) + ./pull-binary-libs.sh + + # build quick using STARR built upstream, as specified by scalaVersion + # (in that sense it's locker, since it was built with starr by that upstream job) + ant -Dstarr.version=$scalaVersion \ + -Dscalac.args.optimise=-optimise \ + -Dlocker.skip=1 -Dstarr.use.released=1 -Dextra.repo.url=$prRepoUrl \ + $testExtraArgs ${testTarget-test.core docs.done} + ;; +esac \ No newline at end of file diff --git a/scripts/readproperties.awk b/scripts/readproperties.awk new file mode 100644 index 0000000000..96da94775b --- /dev/null +++ b/scripts/readproperties.awk @@ -0,0 +1,39 @@ +# Adapted from http://stackoverflow.com/questions/1682442/reading-java-properties-file-from-bash/2318840#2318840 +BEGIN { + FS="="; + n=""; + v=""; + c=0; # Not a line continuation. +} +/^\#/ { # The line is a comment. Breaks line continuation. + c=0; + next; +} +/\\$/ && (c==0) && (NF>=2) { # Name value pair with a line continuation... + e=index($0,"="); + n=substr($0,1,e-1); + v=substr($0,e+1,length($0) - e - 1); # Trim off the backslash. + c=1; # Line continuation mode. + next; +} +/^[^\\]+\\$/ && (c==1) { # Line continuation. Accumulate the value. + v= "" v substr($0,1,length($0)-1); + next; +} +((c==1) || (NF>=2)) && !/^[^\\]+\\$/ { # End of line continuation, or a single line name/value pair + if (c==0) { # Single line name/value pair + e=index($0,"="); + n=substr($0,1,e-1); + v=substr($0,e+1,length($0) - e); + } else { # Line continuation mode - last line of the value. + c=0; # Turn off line continuation mode. + v= "" v $0; + } + # Make sure the name is a legal shell variable name + gsub(/[^A-Za-z0-9_]/,"_",n); + # Silently drop everything that might confuse bash. + gsub(/[\n\r\\\t'"\$!]/,"",v); + print "export " n "=\"" v "\" || echo \"Failed to set " n "\""; # don't make bash crap out when a property could not be parsed + n = ""; + v = ""; +} diff --git a/scripts/repositories-scala-release b/scripts/repositories-scala-release new file mode 100644 index 0000000000..00538a08ff --- /dev/null +++ b/scripts/repositories-scala-release @@ -0,0 +1,7 @@ +[repositories] + plugins: http://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + private-repo: http://private-repo.typesafe.com/typesafe/scala-release-temp/ + typesafe-ivy-releases: http://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly + sbt-plugin-releases: http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + maven-central + local \ No newline at end of file -- cgit v1.2.3