diff options
-rw-r--r-- | scripts/common | 153 | ||||
-rwxr-xr-x | scripts/jobs/integrate/bootstrap (renamed from scripts/jobs/scala-release-2.11.x-build) | 74 | ||||
-rwxr-xr-x | scripts/jobs/integrate/ide | 32 | ||||
-rwxr-xr-x | scripts/jobs/validate/publish-core | 44 | ||||
-rwxr-xr-x | scripts/jobs/validate/test | 17 |
5 files changed, 249 insertions, 71 deletions
diff --git a/scripts/common b/scripts/common new file mode 100644 index 0000000000..b075469379 --- /dev/null +++ b/scripts/common @@ -0,0 +1,153 @@ +# This is for forcibly stopping the job from a subshell (see test +# below). +trap "exit 1" TERM +export TOP_PID=$$ +set -e + +# Known problems : does not fare well with interrupted, partial +# compilations. We should perhaps have a multi-dependency version +# of do_i_have below + +LOGGINGDIR="$WORKSPACE/logs" +mkdir -p $LOGGINGDIR + +unset SBT_HOME +SBT_HOME="$WORKSPACE/.sbt" +mkdir -p $SBT_HOME +IVY_CACHE="$WORKSPACE/.ivy2" +mkdir -p $IVY_CACHE +rm -rf $IVY_CACHE/cache/org.scala-lang + +# temp dir where all 'non-build' operation are performed +TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX) +TMP_DIR="${TMP_ROOT_DIR}/tmp" +mkdir "${TMP_DIR}" + + +# detect sed version and how to enable extended regexes +SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)" + + + +# :docstring test: +# Usage: test <argument ..> +# Executes <argument ..>, logging the launch of the command to the +# main log file, and kills global script execution with the TERM +# signal if the commands ends up failing. +# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES, +# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST +# :end docstring: + +function test() { + echo "### $@" + "$@" + status=$? + if [ $status -ne 0 ]; then + say "### ERROR with $1" + kill -s TERM $TOP_PID + fi +} + +# :docstring say: +# Usage: say <argument ..> +# Prints <argument ..> to both console and the main log file. +# :end docstring: + +function say(){ + (echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log +} + +# General debug logging +# $* - message +function debug () { + echo "----- $*" +} + +function parseScalaProperties(){ + propFile="$baseDir/$1" + if [ ! -f $propFile ]; then + echo "Property file $propFile not found." + exit 1 + else + awk -f "$scriptsDir/readproperties.awk" "$propFile" > "$propFile.sh" + . "$propFile.sh" # yeah yeah, not that secure, improvements welcome (I tried, but bash made me cry again) + fi +} + + +## TAKEN FROM UBER-BUILD, except that it "returns" (via $RES) true/false +# Check if an artifact is available +# $1 - groupId +# $2 - artifacId +# $3 - version +# $4 - extra repository to look in (optional) +# return value in $RES +function checkAvailability () { + pushd "${TMP_DIR}" + rm -rf * + +# pom file for the test project + cat > pom.xml << EOF +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>com.typesafe</groupId> + <artifactId>typesafeDummy</artifactId> + <packaging>war</packaging> + <version>1.0-SNAPSHOT</version> + <name>Dummy</name> + <url>http://127.0.0.1</url> + <dependencies> + <dependency> + <groupId>$1</groupId> + <artifactId>$2</artifactId> + <version>$3</version> + </dependency> + </dependencies> + <repositories> + <repository> + <id>sonatype.snapshot</id> + <name>Sonatype maven snapshot repository</name> + <url>https://oss.sonatype.org/content/repositories/snapshots</url> + <snapshots> + <updatePolicy>daily</updatePolicy> + </snapshots> + </repository> +EOF + + if [ -n "$4" ] + then +# adds the extra repository + cat >> pom.xml << EOF + <repository> + <id>extrarepo</id> + <name>extra repository</name> + <url>$4</url> + </repository> +EOF + fi + + cat >> pom.xml << EOF + </repositories> +</project> +EOF + + set +e + mvn "${MAVEN_ARGS[@]}" compile &> "${TMP_DIR}/mvn.log" + RES=$? + # Quiet the maven, but allow diagnosing problems. + grep -i downloading "${TMP_DIR}/mvn.log" + grep -i exception "${TMP_DIR}/mvn.log" + grep -i error "${TMP_DIR}/mvn.log" + set -e + +# log the result + if [ ${RES} == 0 ] + then + debug "$1:$2:jar:$3 found !" + RES=true + else + debug "$1:$2:jar:$3 not found !" + RES=false + fi + popd +} diff --git a/scripts/jobs/scala-release-2.11.x-build b/scripts/jobs/integrate/bootstrap index 09a97dc34b..46d610018c 100755 --- a/scripts/jobs/scala-release-2.11.x-build +++ b/scripts/jobs/integrate/bootstrap @@ -71,7 +71,6 @@ # set to something besides the default to build nightly snapshots of the modules instead of some tagged version moduleVersioning=${moduleVersioning-"versions.properties"} -baseDir=${WORKSPACE-`pwd`} publishPrivateTask=${publishPrivateTask-"publish"} publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"} publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"} @@ -93,67 +92,11 @@ forceRebuild=${forceRebuild-no} antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak) clean="clean" # TESTING leave empty to speed up testing -scriptsDir="$WORKSPACE/scripts" - -# This is for forcibly stopping the job from a subshell (see test -# below). -trap "exit 1" TERM -export TOP_PID=$$ -set -e - -# Known problems : does not fare well with interrupted, partial -# compilations. We should perhaps have a multi-dependency version -# of do_i_have below - -LOGGINGDIR="$WORKSPACE/logs" -mkdir -p $LOGGINGDIR - -unset SBT_HOME -SBT_HOME="$WORKSPACE/.sbt" -mkdir -p $SBT_HOME -IVY_CACHE="$WORKSPACE/.ivy2" -mkdir -p $IVY_CACHE -rm -rf $IVY_CACHE/cache/org.scala-lang - -# temp dir where all 'non-build' operation are performed -TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX) -TMP_DIR="${TMP_ROOT_DIR}/tmp" -mkdir "${TMP_DIR}" -# detect sed version and how to enable extended regexes -SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)" - - - -# :docstring test: -# Usage: test <argument ..> -# Executes <argument ..>, logging the launch of the command to the -# main log file, and kills global script execution with the TERM -# signal if the commands ends up failing. -# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES, -# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST -# :end docstring: - -function test() { - echo "### $@" - "$@" - status=$? - if [ $status -ne 0 ]; then - say "### ERROR with $1" - kill -s TERM $TOP_PID - fi -} - -# :docstring say: -# Usage: say <argument ..> -# Prints <argument ..> to both console and the main log file. -# :end docstring: - -function say(){ - (echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log -} - +baseDir=${WORKSPACE-`pwd`} +scriptsDir="$baseDir/scripts" +. $scriptsDir/common # we must change ivy home to get a fresh ivy cache, otherwise we get half-bootstrapped scala # rm it in case it existed (and there's no ivy2-shadow, which indicates we're running in a TESTING environment)... @@ -168,17 +111,6 @@ mkdir -p $baseDir/resolutionScratch_ privateCred="private-repo" privateRepo="http://private-repo.typesafe.com/typesafe/scala-release-temp/" -function parseScalaProperties(){ - propFile="$baseDir/$1" - if [ ! -f $propFile ]; then - echo "Property file $propFile not found." - exit 1 - else - awk -f "$scriptsDir/readproperties.awk" "$propFile" > "$propFile.sh" - . "$propFile.sh" # yeah yeah, not that secure, improvements welcome (I tried, but bash made me cry again) - fi -} - ##### git gfxd() { git clean -fxd # TESTING diff --git a/scripts/jobs/integrate/ide b/scripts/jobs/integrate/ide new file mode 100755 index 0000000000..5c1e6199e3 --- /dev/null +++ b/scripts/jobs/integrate/ide @@ -0,0 +1,32 @@ +#!/bin/bash -e +# requires checkout: root is a scala checkout with which to integrate (actually, only required file is versions.properties, as documented below) +# requires env: scalaVersion (specifies binary already built from above checkout), WORKSPACE (provided by jenkins), repo_ref (HEAD of the scala checkout), +# requires files: $baseDir/versions.properties (from checkout -- defines version numbers for modules used to build scala for dbuild...) + +# TODO: remove when integration is up and running +if [ "woele$_scabot_last" != "woele1" ]; then echo "Scabot didn't mark this as last commit -- skipping."; exit 0; fi + +baseDir=${WORKSPACE-`pwd`} +uberBuildUrl=${uberBuildUrl-"https://github.com/scala-ide/uber-build.git"} +uberBuildConfig=${uberBuildConfig-"validator.conf"} # TODO: backport to 2.10.x: uberBuildConfig="validator-2.10.conf" + +uberBuildDir="$baseDir/uber-build/" + +cd $baseDir +if [[ -d $uberBuildDir ]]; then + ( cd $uberBuildDir && git fetch $uberBuildUrl HEAD && git checkout -f FETCH_HEAD && git clean -fxd ) +else + git clone $uberBuildUrl +fi + +echo "maven.version.number=$scalaVersion" >> versions.properties + +# pass prRepoUrl in, which uber-build passes along to dbuild (in sbt-builds-for-ide) +# the "-P pr-scala" maven arg accomplishes the same thing for maven (directly used in uber-build) +BASEDIR="$baseDir" prRepoUrl="$prRepoUrl" MAVEN_ARGS="-P pr-scala"\ + $uberBuildDir/uber-build.sh $uberBuildDir/config/$uberBuildConfig $repo_ref $scalaVersion + +# uber-build puts its local repo under target/m2repo +# wipe the org/scala-lang part, which otherwise just keeps +# growing and growing due to the -$sha-SNAPSHOT approach +[[ -d $baseDir/target/m2repo/org/scala-lang ]] && rm -rf $baseDir/target/m2repo/org/scala-lang diff --git a/scripts/jobs/validate/publish-core b/scripts/jobs/validate/publish-core new file mode 100755 index 0000000000..9dff5a34b0 --- /dev/null +++ b/scripts/jobs/validate/publish-core @@ -0,0 +1,44 @@ +#!/bin/bash -e +# This script publishes the core of Scala to maven for use as locker downstream, +# and saves the relevant properties used in its build artifacts, versions.properties. +# (This means we'll use locker instead of quick downstream in dbuild. +# The only downside is that backend improvements don't improve compiler performance itself until they are in STARR). +# The version is suffixed with "-${sha:0:7}-SNAPSHOT" + +baseDir=${WORKSPACE-`pwd`} +scriptsDir="$baseDir/scripts" +. $scriptsDir/common + +case $prDryRun in + yep) + echo "DRY RUN" + mkdir -p build/pack ; mkdir -p dists/maven/latest + ;; + *) + sha=$(git rev-parse HEAD) # TODO: warn if $repo_ref != $sha (we shouldn't do PR validation using symbolic gitrefs) + echo "sha/repo_ref == $sha/$repo_ref ?" + + parseScalaProperties build.number + + ./pull-binary-libs.sh + # "noyoudont" is there juuuust in case + antDeployArgs="-Dmaven.version.suffix=\"-${sha:0:7}-SNAPSHOT\" -Dremote.snapshot.repository=$prRepoUrl -Drepository.credentials.id=pr-scala -Dremote.release.repository=noyoudont" + + echo ">>> Getting Scala version number." + ant -q $antDeployArgs init + parseScalaProperties buildcharacter.properties # produce maven_version_number + + echo ">>> Checking availability of Scala ${maven_version_number} in $prRepoUrl." + checkAvailability "org.scala-lang" "scala-library" "${maven_version_number}" $prRepoUrl; libraryAvailable=$RES + checkAvailability "org.scala-lang" "scala-reflect" "${maven_version_number}" $prRepoUrl; reflectAvailable=$RES + checkAvailability "org.scala-lang" "scala-compiler" "${maven_version_number}" $prRepoUrl; compilerAvailable=$RES + + if $libraryAvailable && $reflectAvailable && $compilerAvailable; then + echo "Scala core already built!" + else + ant $antDeployArgs $antBuildArgs publish-opt-nodocs + fi + + mv buildcharacter.properties jenkins.properties # parsed by the jenkins job + ;; +esac diff --git a/scripts/jobs/validate/test b/scripts/jobs/validate/test new file mode 100755 index 0000000000..c1c02c80cb --- /dev/null +++ b/scripts/jobs/validate/test @@ -0,0 +1,17 @@ +#!/bin/bash -e + +case $prDryRun in + yep) + echo "DRY RUN" + ;; + *) + ./pull-binary-libs.sh + + # build quick using STARR built upstream, as specified by scalaVersion + # (in that sense it's locker, since it was built with starr by that upstream job) + ant -Dstarr.version=$scalaVersion \ + -Dscalac.args.optimise=-optimise \ + -Dlocker.skip=1 -Dstarr.use.released=1 -Dextra.repo.url=$prRepoUrl \ + $testExtraArgs ${testTarget-test.core docs.done} + ;; +esac
\ No newline at end of file |