diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/common | 20 | ||||
| -rw-r--r-- | scripts/jobs/integrate/bootstrap | 104 | ||||
| -rwxr-xr-x | scripts/jobs/integrate/windows | 22 | ||||
| -rwxr-xr-x | scripts/jobs/validate/publish-core | 7 | ||||
| -rwxr-xr-x | scripts/jobs/validate/test | 16 |
5 files changed, 89 insertions, 80 deletions
diff --git a/scripts/common b/scripts/common index bfddf3d149..95389e5495 100644 --- a/scripts/common +++ b/scripts/common @@ -154,3 +154,23 @@ EOF fi popd } + +# Generate a repositories file with all allowed repositories in our build environment. +# Takes one optional argument, the private repository URL. +# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html +function generateRepositoriesConfig() { + jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} + sbtRepositoryConfig="$scriptsDir/sbt-repositories-config" + echo > "$sbtRepositoryConfig" '[repositories]' + if [ -n "$1" ] + then + echo >> "$sbtRepositoryConfig" " private-repo: $1" + fi + cat >> "$sbtRepositoryConfig" << EOF + jcenter-cache: $jcenterCacheUrl + typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly + sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + maven-central + local +EOF +} diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap index 76673b4f32..4d5dae89a2 100644 --- a/scripts/jobs/integrate/bootstrap +++ b/scripts/jobs/integrate/bootstrap @@ -85,14 +85,12 @@ moduleVersioning=${moduleVersioning-"versions.properties"} publishPrivateTask=${publishPrivateTask-"publish"} -publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"} -publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"} -publishStarrPrivateTask=${publishStarrPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built STARR before) -publishLockerPrivateTask=${publishLockerPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built locker before) +publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"} +publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"} forceRebuild=${forceRebuild-no} -antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak) +sbtBuildTask="testAll" # TESTING leave empty to avoid the sanity check clean="clean" # TESTING leave empty to speed up testing baseDir=${WORKSPACE-`pwd`} @@ -111,21 +109,7 @@ mkdir -p $baseDir/resolutionScratch_ # repo used to publish "locker" scala to (to start the bootstrap) releaseTempRepoCred="private-repo" releaseTempRepoUrl=${releaseTempRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-release-temp/"} -jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} - -# Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs, -# so we need to configure SBT to use these rather than its default, Maven Central. -# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html -sbtRepositoryConfig="$scriptsDir/repositories-scala-release" -cat > "$sbtRepositoryConfig" << EOF -[repositories] - private-repo: $releaseTempRepoUrl - jcenter-cache: $jcenterCacheUrl - typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly - sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] - maven-central - local -EOF +generateRepositoriesConfig $releaseTempRepoUrl ##### git gfxd() { @@ -174,11 +158,11 @@ function st_stagingRepoClose() { # the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir # the new version of sbt-extras ignores sbt-dir, so we pass it in as -Dsbt.global.base # need to set sbt-dir to one that has the gpg.sbt plugin config -sbtArgs="-no-colors -ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13" +sbtArgs="-ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13" sbtBuild() { - echo "### sbtBuild: "$SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" - $SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1 + echo "### sbtBuild: "$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" + $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1 } sbtResolve() { @@ -186,8 +170,8 @@ sbtResolve() { touch build.sbt # Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin used to. cross=${4-binary} - echo "### sbtResolve: $SBT_CMD $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" - $SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" \ + echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" + $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" \ "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \ 'show update' >> $baseDir/logs/resolution 2>&1 } @@ -247,7 +231,7 @@ buildPartest() { else update scala scala-partest "$PARTEST_REF" && gfxd doc="$(docTask $PARTEST_BUILT)" - sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' 'set VersionKeys.scalaCheckVersion := "'$SCALACHECK_VER'"' $clean "$doc" test "${buildTasks[@]}" + sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' $clean "$doc" test "${buildTasks[@]}" PARTEST_BUILT="yes" fi } @@ -282,7 +266,7 @@ buildModules() { buildXML buildParsers buildSwing - buildScalacheck + # buildScalacheck buildPartest } @@ -424,7 +408,7 @@ deriveModuleVersions() { echo "Module versions (versioning strategy: $moduleVersioning):" echo "PARSERS = $PARSERS_VER at $PARSERS_REF" echo "PARTEST = $PARTEST_VER at $PARTEST_REF" - echo "SCALACHECK = $SCALACHECK_VER at $SCALACHECK_REF" + # echo "SCALACHECK = $SCALACHECK_VER at $SCALACHECK_REF" echo "SWING = $SWING_VER at $SWING_REF" echo "XML = $XML_VER at $XML_REF" @@ -444,7 +428,7 @@ removeExistingBuilds() { local storageApiUrl=`echo $releaseTempRepoUrl | sed 's/\(scala-release-temp\)/api\/storage\/\1/'` local scalaLangModules=`curl -s $storageApiUrl/org/scala-lang | jq -r '.children | .[] | "org/scala-lang" + .uri' | grep -v actors-migration` - for module in "org/scalacheck" $scalaLangModules; do + for module in $scalaLangModules; do local artifacts=`curl -s $storageApiUrl/$module | jq -r ".children | .[] | select(.uri | contains(\"$SCALA_VER\")) | .uri"` for artifact in $artifacts; do echo "Deleting $releaseTempRepoUrl$module$artifact" @@ -464,7 +448,7 @@ constructUpdatedModuleVersions() { updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-xml.version.number=$XML_VER") updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dpartest.version.number=$PARTEST_VER") - updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscalacheck.version.number=$SCALACHECK_VER") + # updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscalacheck.version.number=$SCALACHECK_VER") # allow overriding the jline version using a jenkins build parameter if [ ! -z "$JLINE_VER" ] ; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Djline.version=$JLINE_VER"); fi @@ -490,13 +474,7 @@ bootstrap() { git clone --reference $WORKSPACE/.git $WORKSPACE/.git $STARR_DIR cd $STARR_DIR git co $STARR_REF - ant -Dmaven.version.number=$STARR_VER\ - -Dremote.snapshot.repository=NOPE\ - -Dremote.release.repository=$releaseTempRepoUrl\ - -Drepository.credentials.id=$releaseTempRepoCred\ - -Ddocs.skip=1\ - -Dlocker.skip=1\ - $publishStarrPrivateTask >> $baseDir/logs/builds 2>&1 + $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $releaseTempRepoUrl $STARR_VER" $clean publish >> $baseDir/logs/builds 2>&1 ) fi @@ -510,14 +488,7 @@ bootstrap() { # publish more than just core: partest needs scalap # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler if [ ! -z "$STARR_VER" ]; then SET_STARR=-Dstarr.version=$STARR_VER; fi - ant -Dmaven.version.number=$SCALA_VER\ - -Dremote.snapshot.repository=NOPE\ - $SET_STARR\ - -Dremote.release.repository=$releaseTempRepoUrl\ - -Drepository.credentials.id=$releaseTempRepoCred\ - -Ddocs.skip=1\ - -Dlocker.skip=1\ - $publishLockerPrivateTask >> $baseDir/logs/builds 2>&1 + $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $releaseTempRepoUrl $SCALA_VER" $clean publish >> $baseDir/logs/builds 2>&1 echo "### Building modules using locker" @@ -534,7 +505,7 @@ bootstrap() { echo "### Bootstrapping Scala using locker" # # TODO: close all open staging repos so that we can be reasonably sure the only open one we see after publishing below is ours - # # the ant call will create a new one + # # the sbt call will create a new one # # Rebuild Scala with these modules so that all binary versions are consistent. # Update versions.properties to new modules. @@ -542,23 +513,35 @@ bootstrap() { # don't skip locker (-Dlocker.skip=1), or stability will fail # overwrite "locker" version of scala at private-repo with bootstrapped version cd $baseDir - rm -rf build/ # must leave everything else in $baseDir for downstream jobs + rm -rf build/ - ant -Dstarr.version=$SCALA_VER\ - -Dextra.repo.url=$releaseTempRepoUrl\ - -Dmaven.version.suffix=$SCALA_VER_SUFFIX\ + $SBT_CMD $sbtArgs \ + --warn \ + -Dstarr.version=$SCALA_VER \ ${updatedModuleVersions[@]} \ - -Dupdate.versions=1\ - -Dscaladoc.git.commit=$SCALADOC_SOURCE_LINKS_VER\ - -Dremote.snapshot.repository=NOPE\ - -Dremote.release.repository=$releaseTempRepoUrl\ - -Drepository.credentials.id=$releaseTempRepoCred\ - -Dscalac.args.optimise=-opt:l:classpath\ - $antBuildTask $publishPrivateTask + "setupBootstrapQuick $releaseTempRepoUrl $SCALA_VER" \ + $clean \ + $sbtBuildTask \ + dist/mkQuick \ + publish | grep -v "was too long to be displayed in the webview, and will be left out" # clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala rm -rf $baseDir/ivy2 + # Run stability tests using the just built version as "quick" and a new version as "strap" + mv build/quick quick1 + rm -rf build/ + $SBT_CMD $sbtArgs \ + --warn \ + -Dstarr.version=$SCALA_VER \ + ${updatedModuleVersions[@]} \ + "setupBootstrapQuick $releaseTempRepoUrl $SCALA_VER" \ + $clean \ + dist/mkQuick + mv build/quick build/strap + mv quick1 build/quick + tools/stability-test.sh + # TODO: create PR with following commit (note that release will have been tagged already) # git commit versions.properties -m"Bump versions.properties for $SCALA_VER." } @@ -571,7 +554,12 @@ publishSonatype() { # stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary, # since we're just publishing an existing build echo "### Publishing core to sonatype" - ant -Dmaven.version.number=$SCALA_VER $publishSonatypeTaskCore + $SBT_CMD $sbtArgs \ + --warn \ + -Dstarr.version=$SCALA_VER \ + ${updatedModuleVersions[@]} \ + "setupBootstrapPublish $releaseTempRepoUrl $SCALA_VER" \ + $publishSonatypeTaskCore | grep -v "was too long to be displayed in the webview, and will be left out" echo "### Publishing modules to sonatype" # build/test/publish scala core modules to sonatype (this will start a new staging repo) diff --git a/scripts/jobs/integrate/windows b/scripts/jobs/integrate/windows index ba48c5bc25..5e04b0b380 100755 --- a/scripts/jobs/integrate/windows +++ b/scripts/jobs/integrate/windows @@ -1,15 +1,19 @@ #!/bin/bash -./pull-binary-libs.sh - -export ANT_OPTS="-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M" - -# TODO: don't hardcode these paths -- should be in scala/scala-jenkins-infra, passed in through env vars from jenkins -export PATH='/cygdrive/c/Program Files/Java/jdk1.8.0_92/bin:/cygdrive/c/apache-ant-1.9.6/bin:/cygdrive/c/Program Files (x86)/Git-2.5.3/Cmd:/bin:/usr/bin:' -export JAVA_HOME='C:/Program Files/Java/jdk1.8.0_92' +baseDir=${WORKSPACE-`pwd`} +scriptsDir="$baseDir/scripts" +. $scriptsDir/common java -version javac -version -ant -version -ant test-opt +generateRepositoriesConfig + +SBT="java $JAVA_OPTS -Dsbt.ivy.home=$WORKSPACE/.ivy2 -jar $sbtLauncher -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig" + +# Build locker with STARR +$SBT --warn "setupPublishCore" generateBuildCharacterPropertiesFile publishLocal + +# Build quick and run the tests +parseScalaProperties buildcharacter.properties +$SBT -Dstarr.version=$maven_version_number --warn "setupValidateTest" testAll | grep -v "was too long to be displayed in the webview, and will be left out" diff --git a/scripts/jobs/validate/publish-core b/scripts/jobs/validate/publish-core index b0bfd48083..c71fbd12b7 100755 --- a/scripts/jobs/validate/publish-core +++ b/scripts/jobs/validate/publish-core @@ -9,6 +9,9 @@ baseDir=${WORKSPACE-`pwd`} scriptsDir="$baseDir/scripts" . $scriptsDir/common +generateRepositoriesConfig $prRepoUrl +SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig" + case $prDryRun in yep) echo "DRY RUN" @@ -16,7 +19,7 @@ case $prDryRun in ;; *) echo ">>> Getting Scala version number." - $SBT_CMD --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile + $SBT --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile parseScalaProperties buildcharacter.properties # produce maven_version_number echo ">>> Checking availability of Scala ${maven_version_number} in $prRepoUrl." @@ -27,7 +30,7 @@ case $prDryRun in if $libraryAvailable && $reflectAvailable && $compilerAvailable; then echo "Scala core already built!" else - $SBT_CMD --warn "setupPublishCore $prRepoUrl" publish + $SBT --warn "setupPublishCore $prRepoUrl" publish fi mv buildcharacter.properties jenkins.properties # parsed by the jenkins job diff --git a/scripts/jobs/validate/test b/scripts/jobs/validate/test index 3cd8af5608..9938319dd8 100755 --- a/scripts/jobs/validate/test +++ b/scripts/jobs/validate/test @@ -4,6 +4,9 @@ baseDir=${WORKSPACE-`pwd`} scriptsDir="$baseDir/scripts" . $scriptsDir/common +generateRepositoriesConfig $prRepoUrl +SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig" + case $prDryRun in yep) @@ -15,21 +18,12 @@ case $prDryRun in # build quick using STARR built upstream, as specified by scalaVersion # (in that sense it's locker, since it was built with starr by that upstream job); # and run JUnit tests, partest, OSGi tests, MiMa and scaladoc - $SBT_CMD \ + $SBT \ -Dstarr.version=$scalaVersion \ --warn \ "setupValidateTest $prRepoUrl" \ $testExtraArgs \ - "test" \ - "partest run pos neg jvm" \ - "partest res scalap specialized scalacheck" \ - "partest instrumented presentation" \ - "partest --srcpath scaladoc" \ - osgiTestFelix/test \ - osgiTestEclipse/test \ - library/mima \ - reflect/mima \ - doc + testAll | grep -v "was too long to be displayed in the webview, and will be left out" ;; |