diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/common | 23 | ||||
| -rwxr-xr-x[-rw-r--r--] | scripts/jobs/integrate/bootstrap | 351 | ||||
| -rwxr-xr-x | scripts/jobs/integrate/ide | 3 | ||||
| -rwxr-xr-x | scripts/jobs/integrate/windows | 27 | ||||
| -rwxr-xr-x | scripts/jobs/validate/publish-core | 7 | ||||
| -rwxr-xr-x | scripts/jobs/validate/test | 18 | ||||
| -rwxr-xr-x | scripts/stability-test.sh | 29 |
7 files changed, 199 insertions, 259 deletions
diff --git a/scripts/common b/scripts/common index f2202b9165..c68a80fd74 100644 --- a/scripts/common +++ b/scripts/common @@ -19,7 +19,7 @@ mkdir -p $IVY_CACHE rm -rf $IVY_CACHE/cache/org.scala-lang SBT_CMD=${sbtCmd-sbt} -SBT_CMD="$SBT_CMD -sbt-version 0.13.12" +SBT_CMD="$SBT_CMD -sbt-version 0.13.13" # temp dir where all 'non-build' operation are performed TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX) @@ -154,3 +154,24 @@ EOF fi popd } + +# Generate a repositories file with all allowed repositories in our build environment. +# Takes a variable number of additional repositories as argument. +# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html +function generateRepositoriesConfig() { + jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} + sbtRepositoryConfig="$scriptsDir/sbt-repositories-config" + echo > "$sbtRepositoryConfig" '[repositories]' + if [[ $# -gt 0 ]]; then + for i in $(seq 1 $#); do + echo >> "$sbtRepositoryConfig" " script-repo-$i: ${!i}" + done + fi + cat >> "$sbtRepositoryConfig" << EOF + jcenter-cache: $jcenterCacheUrl + typesafe-ivy-releases: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly + sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + maven-central + local +EOF +} diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap index 85be794276..65c8ef5551 100644..100755 --- a/scripts/jobs/integrate/bootstrap +++ b/scripts/jobs/integrate/bootstrap @@ -5,13 +5,10 @@ # - determine module versions # - build minimal core (aka locker) of Scala, use the determined version number, publish to scala-integration # - build those modules where a binary compatible version doesn't exist, publish to scala-integration -# - build Scala using the previously built core and bootstrap modules, publish to scala-integration, overwriting -# the existing artifacts +# - build Scala using the previously built core and bootstrap modules, publish to scala-integration # - for releases # - stage Scala on sonatype # - rebuild modules that needed a rebuild with this Scala build, and stage them on sonatype -# - for nightlies -# - force rebuild all modules and publish them locally (for testing purposes) # - the Scala version is serialized to jenkins.properties, which is passed downstream to scala-release jobs @@ -21,21 +18,16 @@ # - Or have the current HEAD tagged as v$base$suffix # - To prevent staging on sonatype (for testing), set publishToSonatype to anything but "yes" # - Note: After building a release, the jenkins job provides an updated versions.properties file as artifact. -# Put this file in the Scala repo and create a pull request, and also update the file build.number. +# Put this file in the Scala repo and create a pull request, also update `baseVersion in Global` in build.sbt. # # - Otherwise, an integration build is performed: -# - version number is read from the build.number file, extended with -[bin|pre]-$sha +# - version number is read from the build.sbt, extended with -[bin|pre]-$sha -# Specifying module versions: there are two modes -# - If moduleVersioning="versions.properties" (default): in this mode we use release versions for the modules. -# - Module versions are read from the versions.properties file. -# - Set <MODULE>_VER to override the default, e.g. XML_VER="1.0.4". -# - The git revision is set to <MODULE>_REF="v$<MODULE>_VER". Make sure the tag exists (you can't override <MODULE>_REF). -# -# - Otherwise (moduleVersioning has some other value): in this mode we use nightly version numbers for modules. -# - By default the script sets all <MODULE>_REF to "HEAD", override to build a specific revision. -# - The <MODULE>_VER is set to a nightly version, for example "1.0.3-7-g14888a2-nightly" (you can't override <MODULE>_VER) +# Specifying module versions. We use release versions for modules. +# - Module versions are read from the versions.properties file. +# - Set <MODULE>_VER to override the default, e.g. XML_VER="1.0.4". +# - The git revision is set to <MODULE>_REF="v$<MODULE>_VER". Make sure the tag exists (you can't override <MODULE>_REF). # Modules are automatically built if necessary. @@ -82,16 +74,15 @@ # Note: private-repo used to be private-repo.typesafe.com. now we're running artifactory on scala-ci.typesafe.com/artifactory -moduleVersioning=${moduleVersioning-"versions.properties"} - publishPrivateTask=${publishPrivateTask-"publish"} -publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"} -publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"} -publishLockerPrivateTask=${publishLockerPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built locker before) +publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"} +publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"} forceRebuild=${forceRebuild-no} -antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak) +sbtBuildTask=${sbtBuildTask-"testAll"} # TESTING leave empty to avoid the sanity check +testStability=${testStability-yes} + clean="clean" # TESTING leave empty to speed up testing baseDir=${WORKSPACE-`pwd`} @@ -107,31 +98,17 @@ mkdir -p $baseDir/ivy2 rm -rf $baseDir/resolutionScratch_ mkdir -p $baseDir/resolutionScratch_ -function generateRepositoriesConfig(){ - # Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs, - # so we need to configure SBT to use these rather than its default, Maven Central. - # See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html - sbtRepositoryConfig="$scriptsDir/repositories-scala-release" - jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} - cat > "$sbtRepositoryConfig" << EOF -[repositories] - script-repo: $1 - jcenter-cache: $jcenterCacheUrl - typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly - sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] - maven-central - local -EOF -} - -integrationRepoCred="private-repo" - -# repo for locker, quick and the modules +# repo to publish builds integrationRepoUrl=${integrationRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-integration/"} -# adding `integrationRepoUrl` to find the locker scala version when building modules generateRepositoriesConfig $integrationRepoUrl +# ARGH trying to get this to work on multiple versions of sbt-extras... +# the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir +# the new version of sbt-extras ignores sbt-dir, so we pass it in as -Dsbt.global.base +# need to set sbt-dir to one that has the gpg.sbt plugin config +sbtArgs="-ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13" + ##### git gfxd() { git clean -fxd # TESTING @@ -174,31 +151,25 @@ function st_stagingRepoClose() { echo "{\"data\":{\"description\":\"$message\",\"stagedRepositoryIds\":[\"$repo\"]}}" | st_curl -X POST -d @- "$stApi/staging/bulk/close" } - -# ARGH trying to get this to work on multiple versions of sbt-extras... -# the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir -# the new version of sbt-extras ignores sbt-dir, so we pass it in as -Dsbt.global.base -# need to set sbt-dir to one that has the gpg.sbt plugin config -sbtArgs="-no-colors -ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13" +#### sbt tools sbtBuild() { - echo "### sbtBuild: "$SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" - $SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1 + echo "### sbtBuild: "$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" + $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1 } sbtResolve() { cd $baseDir/resolutionScratch_ touch build.sbt - # Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin. + # Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin used to. cross=${4-binary} - echo "### sbtResolve: $SBT_CMD $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" - $SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" \ + echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" + $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" \ "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \ 'show update' >> $baseDir/logs/resolution 2>&1 } -# Oh boy... can't use scaladoc to document scala-xml/scala-parser-combinators -# if scaladoc depends on the same version of scala-xml/scala-parser-combinators. +# Oh boy... can't use scaladoc to document scala-xml if scaladoc depends on the same version of scala-xml. # Even if that version is available through the project's resolvers, sbt won't look past this project. # SOOOOO, we set the version to a dummy (-DOC), generate documentation, # then set the version to the right one and publish (which won't re-gen the docs). @@ -214,100 +185,56 @@ sbtResolve() { # scala-xml depends on scala-library, so sbt tries to find the scala-library of the version that we are currently building, # which exists only in artifactory. +docTask() { + if [[ "$STARR_REF" != "" && "$1" != "yes" ]]; then + # Don't build module docs on the first round of module builds when bootstrapping + # a binary incompatible compiler change to avoid linkage errors with using the old Scaladoc + echo set publishArtifact in packageDoc in Compile := false + else + echo doc + fi +} + buildXML() { if [ "$XML_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-xml" $XML_VER ) then echo "Found scala-xml $XML_VER; not building." else update scala scala-xml "$XML_REF" && gfxd - sbtBuild 'set version := "'$XML_VER'-DOC"' $clean doc 'set version := "'$XML_VER'"' test "${buildTasks[@]}" + doc="$(docTask $XML_BUILT)" + sbtBuild 'set version := "'$XML_VER'-DOC"' $clean "$doc" 'set version := "'$XML_VER'"' test "${buildTasks[@]}" XML_BUILT="yes" # ensure the module is built and published when buildXML is invoked for the second time, see comment above fi } -buildParsers() { - if [ "$PARSERS_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-parser-combinators" $PARSERS_VER ) - then echo "Found scala-parser-combinators $PARSERS_VER; not building." - else - update scala scala-parser-combinators "$PARSERS_REF" && gfxd - sbtBuild 'set version := "'$PARSERS_VER'-DOC"' $clean doc 'set version := "'$PARSERS_VER'"' test "${buildTasks[@]}" - PARSERS_BUILT="yes" - fi -} - buildPartest() { if [ "$PARTEST_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-partest" $PARTEST_VER ) then echo "Found scala-partest $PARTEST_VER; not building." else update scala scala-partest "$PARTEST_REF" && gfxd - sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' 'set VersionKeys.scalaCheckVersion := "'$SCALACHECK_VER'"' $clean test "${buildTasks[@]}" + doc="$(docTask $PARTEST_BUILT)" + sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' $clean "$doc" test "${buildTasks[@]}" PARTEST_BUILT="yes" fi } -buildContinuations() { - if [ "$CONT_PLUG_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.plugins" "scala-continuations-plugin" $CONTINUATIONS_VER full ) - then echo "Found scala-continuations-plugin $CONTINUATIONS_VER; not building." - else - update scala scala-continuations $CONTINUATIONS_REF && gfxd - - $SBT_CMD $sbtArgs 'project plugin' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ - 'set version := "'$CONTINUATIONS_VER'"' $clean "compile:package" test "${buildTasks[@]}" # https://github.com/scala/scala-continuations/pull/4 - CONT_PLUG_BUILT="yes" - fi - - if [ "$CONT_LIB_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.plugins" "scala-continuations-library" $CONTINUATIONS_VER ) - then echo "Found scala-continuations-library $CONTINUATIONS_VER; not building." - else - update scala scala-continuations $CONTINUATIONS_REF && gfxd - $SBT_CMD $sbtArgs 'project library' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ - 'set version := "'$CONTINUATIONS_VER'"' $clean test "${buildTasks[@]}" - CONT_LIB_BUILT="yes" - fi -} - -buildSwing() { - if [ "$SWING_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-swing" $SWING_VER ) - then echo "Found scala-swing $SWING_VER; not building." - else - update scala scala-swing "$SWING_REF" && gfxd - sbtBuild 'set version := "'$SWING_VER'"' $clean test "${buildTasks[@]}" - SWING_BUILT="yes" - fi -} - -buildActorsMigration(){ - if [ "$ACTORS_MIGRATION_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang" "scala-actors-migration" $ACTORS_MIGRATION_VER ) - then echo "Found scala-actors-migration $ACTORS_MIGRATION_VER; not building." - else - update scala actors-migration "$ACTORS_MIGRATION_REF" && gfxd - # not running tests because - # [error] Test scala.actors.migration.NestedReact.testNestedReactAkka failed: java.util.concurrent.TimeoutException: Futures timed out after [20 seconds] - sbtBuild 'set version := "'$ACTORS_MIGRATION_VER'"' 'set VersionKeys.continuationsVersion := "'$CONTINUATIONS_VER'"' $clean "${buildTasks[@]}" - ACTORS_MIGRATION_BUILT="yes" - fi -} - # should only be called with publishTasks publishing to artifactory -buildScalacheck(){ +buildScalaCheck(){ if [ "$SCALACHECK_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scalacheck" "scalacheck" $SCALACHECK_VER ) then echo "Found scalacheck $SCALACHECK_VER; not building." else update rickynils scalacheck $SCALACHECK_REF && gfxd - sbtBuild 'set version := "'$SCALACHECK_VER'"' 'set VersionKeys.scalaParserCombinatorsVersion := "'$PARSERS_VER'"' $clean publish # test times out NOTE: never published to sonatype + doc="$(docTask $SCALACHECK_BUILT)" + sbtBuild 'set version := "'$SCALACHECK_VER'"' 'set VersionKeys.scalaParserCombinatorsVersion := "'$PARSERS_VER'"' $clean "$doc" publish # test times out NOTE: never published to sonatype SCALACHECK_BUILT="yes" fi } -# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to artifactory) +# build modules, using ${buildTasks[@]} (except for ScalaCheck, which is hard-coded to publish to artifactory) buildModules() { publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")") buildTasks=($publishPrivateTask) buildXML - buildParsers - buildContinuations - buildSwing - buildActorsMigration - buildScalacheck + # buildScalaCheck buildPartest } @@ -315,10 +242,6 @@ buildPublishedModules() { publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)") buildTasks=($publishSonatypeTaskModules) buildXML - buildParsers - buildContinuations - buildSwing - buildActorsMigration buildPartest } @@ -357,10 +280,8 @@ scalaVerToBinary() { determineScalaVersion() { cd $WORKSPACE parseScalaProperties "versions.properties" - echo "repo_ref=2.11.x" >> $baseDir/jenkins.properties # for the -dist downstream jobs that build the actual archives - - # each of the branches below defines the following vars: SCALA_VER_BASE, SCALA_VER_SUFFIX, SCALADOC_SOURCE_LINKS_VER, publishToSonatype + # each of the branches below defines the following vars: SCALA_VER_BASE, SCALA_VER_SUFFIX, publishToSonatype if [ -z "$SCALA_VER_BASE" ]; then echo "No SCALA_VER_BASE specified." @@ -369,15 +290,10 @@ determineScalaVersion() { if [ -z "$scalaTag" ] then echo "No tag found, running an integration build." - parseScalaProperties "build.number" - SCALA_VER_BASE="$version_major.$version_minor.$version_patch" - local shaSuffix=$(git rev-parse HEAD | cut -c1-7) - local cross="bin" - if [[ $SCALA_VER_BASE =~ ^.*\.0$ ]]; then - cross="pre" - fi - SCALA_VER_SUFFIX="-$cross-$shaSuffix" - SCALADOC_SOURCE_LINKS_VER=$(git rev-parse HEAD) + $SBT_CMD $sbtArgs 'set baseVersionSuffix in Global := "SHA"' generateBuildCharacterPropertiesFile + parseScalaProperties "buildcharacter.properties" + SCALA_VER_BASE="$maven_version_base" + SCALA_VER_SUFFIX="$maven_version_suffix" # TODO: publish nightly snapshot using this script - currently it's a separate jenkins job still running at EPFL. publishToSonatype="no" @@ -387,7 +303,6 @@ determineScalaVersion() { local RE='v*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' # don't change this to make it more accurate, it's not worth it SCALA_VER_BASE="$(echo $scalaTag | sed -e "s#$RE#\1.\2.\3#")" SCALA_VER_SUFFIX="$(echo $scalaTag | sed -e "s#$RE#\4#")" - SCALADOC_SOURCE_LINKS_VER=$scalaTag if [ "$SCALA_VER_BASE" == "$scalaTag" ]; then echo "Could not parse version $scalaTag" @@ -397,8 +312,6 @@ determineScalaVersion() { fi else publishToSonatype=${publishToSonatype-"yes"} # unless forced previously, publish - # if version base/suffix are provided, we assume a corresponding tag exists for the scaladoc source links - SCALADOC_SOURCE_LINKS_VER="v$SCALA_VER_BASE$SCALA_VER_SUFFIX" fi SCALA_VER="$SCALA_VER_BASE$SCALA_VER_SUFFIX" @@ -412,61 +325,18 @@ determineScalaVersion() { echo "Building Scala $SCALA_VER." } -deriveVersion() { - update $1 $2 $3 &> /dev/null - echo "$(git describe --tag --match=v* | cut -dv -f2)-nightly" -} - -deriveVersionAnyTag() { - update $1 $2 $3 &> /dev/null - echo "$(git describe --tag | cut -dv -f2)-nightly" -} - -# determineScalaVersion must have been called +# determineScalaVersion must have been called (versions.properties is parsed to env vars) deriveModuleVersions() { - if [ "$moduleVersioning" == "versions.properties" ]; then - # use versions.properties as defaults when no version specified on the command line - XML_VER=${XML_VER-$scala_xml_version_number} - PARSERS_VER=${PARSERS_VER-$scala_parser_combinators_version_number} - CONTINUATIONS_VER=${CONTINUATIONS_VER-$scala_continuations_plugin_version_number} - SWING_VER=${SWING_VER-$scala_swing_version_number} - ACTORS_MIGRATION_VER=${ACTORS_MIGRATION_VER-$actors_migration_version_number} - PARTEST_VER=${PARTEST_VER-$partest_version_number} - SCALACHECK_VER=${SCALACHECK_VER-$scalacheck_version_number} - - XML_REF="v$XML_VER" - PARSERS_REF="v$PARSERS_VER" - CONTINUATIONS_REF="v$CONTINUATIONS_VER" - SWING_REF="v$SWING_VER" - ACTORS_MIGRATION_REF="v$ACTORS_MIGRATION_VER" - PARTEST_REF="v$PARTEST_VER" - SCALACHECK_REF="$SCALACHECK_VER" # no `v` in their tags - else - # use HEAD as default when no revision is specified on the command line - XML_REF=${XML_REF-"HEAD"} - PARSERS_REF=${PARSERS_REF-"HEAD"} - CONTINUATIONS_REF=${CONTINUATIONS_REF-"HEAD"} - SWING_REF=${SWING_REF-"HEAD"} - ACTORS_MIGRATION_REF=${ACTORS_MIGRATION_REF-"HEAD"} - PARTEST_REF=${PARTEST_REF-"HEAD"} - SCALACHECK_REF=${SCALACHECK_REF-"HEAD"} - - XML_VER=$(deriveVersion scala scala-xml "$XML_REF") - PARSERS_VER=$(deriveVersion scala scala-parser-combinators "$PARSERS_REF") - CONTINUATIONS_VER=$(deriveVersion scala scala-continuations "$CONTINUATIONS_REF") - SWING_VER=$(deriveVersion scala scala-swing "$SWING_REF") - ACTORS_MIGRATION_VER=$(deriveVersion scala actors-migration "$ACTORS_MIGRATION_REF") - PARTEST_VER=$(deriveVersion scala scala-partest "$PARTEST_REF") - SCALACHECK_VER=$(deriveVersionAnyTag rickynils scalacheck "$SCALACHECK_REF") - fi + XML_VER=${XML_VER-$scala_xml_version_number} + PARTEST_VER=${PARTEST_VER-$partest_version_number} + SCALACHECK_VER=${SCALACHECK_VER-$scalacheck_version_number} + + XML_REF="v$XML_VER" + PARTEST_REF="v$PARTEST_VER" + SCALACHECK_REF="$SCALACHECK_VER" # no `v` in their tags - echo "Module versions (versioning strategy: $moduleVersioning):" - echo "ACTORS_MIGRATION = $ACTORS_MIGRATION_VER at $ACTORS_MIGRATION_REF" - echo "CONTINUATIONS = $CONTINUATIONS_VER at $CONTINUATIONS_REF" - echo "PARSERS = $PARSERS_VER at $PARSERS_REF" echo "PARTEST = $PARTEST_VER at $PARTEST_REF" - echo "SCALACHECK = $SCALACHECK_VER at $SCALACHECK_REF" - echo "SWING = $SWING_VER at $SWING_REF" + # echo "SCALACHECK = $SCALACHECK_VER at $SCALACHECK_REF" echo "XML = $XML_VER at $XML_REF" } @@ -508,21 +378,13 @@ removeExistingBuilds() { constructUpdatedModuleVersions() { updatedModuleVersions=() - # force the new module versions for building the core. these may be different from the values in versions.properties, - # either because the variables (XML_VER) were provided, or because we're building the modules from HEAD. - # in the common case, the values are the same as in versions.properties. - updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dactors-migration.version.number=$ACTORS_MIGRATION_VER") - updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-continuations-library.version.number=$CONTINUATIONS_VER") - updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-continuations-plugin.version.number=$CONTINUATIONS_VER") - updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-parser-combinators.version.number=$PARSERS_VER") - updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-swing.version.number=$SWING_VER") + # force the new module versions for building the core. these may be different from the values in versions.properties + # if the variables (XML_VER) were provided. in the common case, the values are the same as in versions.properties. updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-xml.version.number=$XML_VER") - updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dpartest.version.number=$PARTEST_VER") - updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscalacheck.version.number=$SCALACHECK_VER") + # updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscalacheck.version.number=$SCALACHECK_VER") - # allow overriding the akka-actors and jline version using a jenkins build parameter - if [ ! -z "$AKKA_ACTOR_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dakka-actor.version.number=$AKKA_ACTOR_VER"); fi + # allow overriding the jline version using a jenkins build parameter if [ ! -z "$JLINE_VER" ] ; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Djline.version=$JLINE_VER"); fi if [ ! -z "$SCALA_BINARY_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala.binary.version=$SCALA_BINARY_VER"); fi @@ -534,6 +396,22 @@ bootstrap() { cd $WORKSPACE + #### (Optional) STARR. + if [ ! -z "$STARR_REF" ]; then + echo "### Building STARR" + + STARR_DIR=./scala-starr + STARR_VER_SUFFIX="-$(git rev-parse --short $STARR_REF)-starr" + STARR_VER=$SCALA_VER_BASE$STARR_VER_SUFFIX + rm -rf "$STARR_DIR" + ( + git clone --reference $WORKSPACE/.git $WORKSPACE/.git $STARR_DIR + cd $STARR_DIR + git co $STARR_REF + $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $integrationRepoUrl $STARR_VER" $clean publish >> $baseDir/logs/builds 2>&1 + ) + fi + #### LOCKER echo "### Building locker" @@ -543,15 +421,8 @@ bootstrap() { # must publish under $SCALA_VER so that the modules will depend on this (binary) version of Scala # publish more than just core: partest needs scalap # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler - ant -Dmaven.version.number=$SCALA_VER\ - -Dremote.snapshot.repository=NOPE\ - -Dremote.release.repository=$integrationRepoUrl\ - -Drepository.credentials.id=$integrationRepoCred\ - -Dscalac.args.optimise=-optimise\ - -Ddocs.skip=1\ - -Dlocker.skip=1\ - $publishLockerPrivateTask >> $baseDir/logs/builds 2>&1 - + if [ ! -z "$STARR_VER" ]; then SET_STARR=-Dstarr.version=$STARR_VER; fi + $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $integrationRepoUrl $SCALA_VER" $clean publish >> $baseDir/logs/builds 2>&1 echo "### Building modules using locker" @@ -568,30 +439,23 @@ bootstrap() { echo "### Bootstrapping Scala using locker" # # TODO: close all open staging repos so that we can be reasonably sure the only open one we see after publishing below is ours - # # the ant call will create a new one + # # the sbt call will create a new one # # Rebuild Scala with these modules so that all binary versions are consistent. # Update versions.properties to new modules. # Sanity check: make sure the Scala test suite passes / docs can be generated with these modules. - # don't skip locker (-Dlocker.skip=1), or stability will fail - # overwrite "locker" version of scala at private-repo with bootstrapped version cd $baseDir - rm -rf build/ # must leave everything else in $baseDir for downstream jobs - - # scala.full.version determines the dependency of scala-dist on the continuations plugin, - # which is fully cross-versioned (for $SCALA_VER, the version we're releasing) - ant -Dstarr.version=$SCALA_VER\ - -Dscala.full.version=$SCALA_VER\ - -Dextra.repo.url=$integrationRepoUrl\ - -Dmaven.version.suffix=$SCALA_VER_SUFFIX\ + rm -rf build/ + + $SBT_CMD $sbtArgs \ + --warn \ + -Dstarr.version=$SCALA_VER \ ${updatedModuleVersions[@]} \ - -Dupdate.versions=1\ - -Dscaladoc.git.commit=$SCALADOC_SOURCE_LINKS_VER\ - -Dremote.snapshot.repository=NOPE\ - -Dremote.release.repository=$integrationRepoUrl\ - -Drepository.credentials.id=$integrationRepoCred\ - -Dscalac.args.optimise=-optimise\ - $antBuildTask $publishPrivateTask + "setupBootstrapQuick $integrationRepoUrl $SCALA_VER" \ + $clean \ + $sbtBuildTask \ + dist/mkQuick \ + publish # clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala rm -rf $baseDir/ivy2 @@ -600,6 +464,26 @@ bootstrap() { # git commit versions.properties -m"Bump versions.properties for $SCALA_VER." } +testStability() { + echo "### Testing stability" + + cd $baseDir + + # Run stability tests using the just built version as "quick" and a new version as "strap" + mv build/quick quick1 + rm -rf build/ + $SBT_CMD $sbtArgs \ + --warn \ + -Dstarr.version=$SCALA_VER \ + ${updatedModuleVersions[@]} \ + "setupBootstrapQuick $integrationRepoUrl $SCALA_VER" \ + $clean \ + dist/mkQuick + mv build/quick build/strap + mv quick1 build/quick + $scriptsDir/stability-test.sh +} + # assumes we just bootstrapped, and current directory is $baseDir # publishes locker to sonatype, then builds modules again (those for which version numbers were provided), # and publishes those to sonatype as well @@ -608,7 +492,12 @@ publishSonatype() { # stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary, # since we're just publishing an existing build echo "### Publishing core to sonatype" - ant -Dmaven.version.number=$SCALA_VER $publishSonatypeTaskCore + $SBT_CMD $sbtArgs \ + --warn \ + -Dstarr.version=$SCALA_VER \ + ${updatedModuleVersions[@]} \ + "setupBootstrapPublish $integrationRepoUrl $SCALA_VER" \ + $publishSonatypeTaskCore echo "### Publishing modules to sonatype" # build/test/publish scala core modules to sonatype (this will start a new staging repo) @@ -639,6 +528,10 @@ removeExistingBuilds $integrationRepoUrl bootstrap +if [ "$testStability" == "yes" ] + then testStability +fi + if [ "$publishToSonatype" == "yes" ] then publishSonatype fi diff --git a/scripts/jobs/integrate/ide b/scripts/jobs/integrate/ide index ea7d746822..c39facbc3d 100755 --- a/scripts/jobs/integrate/ide +++ b/scripts/jobs/integrate/ide @@ -3,8 +3,7 @@ # requires env: scalaVersion (specifies binary already built from above checkout), WORKSPACE (provided by jenkins), repo_ref (HEAD of the scala checkout), # requires files: $baseDir/versions.properties (from checkout -- defines version numbers for modules used to build scala for dbuild...) -echo "IDE integration disabled for now on 2.11.x. Punting." -echo "see https://github.com/scala/scala-dev/issues/104" +echo "IDE integration not yet available on 2.12.x. Punting." exit 0 # TODO: remove when integration is up and running diff --git a/scripts/jobs/integrate/windows b/scripts/jobs/integrate/windows index baed9f6236..f5e068684e 100755 --- a/scripts/jobs/integrate/windows +++ b/scripts/jobs/integrate/windows @@ -1,18 +1,19 @@ -#!/bin/bash -x +#!/bin/bash -./pull-binary-libs.sh - -export ANT_OPTS="-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M" - -export JAVA_HOME="C:/java/jdk-1.6" -export PATH="$(cygpath $JAVA_HOME)/bin:$PATH" +baseDir=${WORKSPACE-`pwd`} +scriptsDir="$baseDir/scripts" +. $scriptsDir/common java -version javac -version -ant -version -ant \ - -Dstarr.version=2.11.10 \ - -Dscalac.args.optimise=-optimise \ - -Dlocker.skip=1 \ - test +generateRepositoriesConfig + +SBT="java $JAVA_OPTS -Dsbt.ivy.home=$WORKSPACE/.ivy2 -jar $sbtLauncher -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig" + +# Build locker with STARR +$SBT --warn "setupPublishCore" generateBuildCharacterPropertiesFile publishLocal + +# Build quick and run the tests +parseScalaProperties buildcharacter.properties +$SBT -Dstarr.version=$maven_version_number --warn "setupValidateTest" testAll diff --git a/scripts/jobs/validate/publish-core b/scripts/jobs/validate/publish-core index b0bfd48083..c71fbd12b7 100755 --- a/scripts/jobs/validate/publish-core +++ b/scripts/jobs/validate/publish-core @@ -9,6 +9,9 @@ baseDir=${WORKSPACE-`pwd`} scriptsDir="$baseDir/scripts" . $scriptsDir/common +generateRepositoriesConfig $prRepoUrl +SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig" + case $prDryRun in yep) echo "DRY RUN" @@ -16,7 +19,7 @@ case $prDryRun in ;; *) echo ">>> Getting Scala version number." - $SBT_CMD --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile + $SBT --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile parseScalaProperties buildcharacter.properties # produce maven_version_number echo ">>> Checking availability of Scala ${maven_version_number} in $prRepoUrl." @@ -27,7 +30,7 @@ case $prDryRun in if $libraryAvailable && $reflectAvailable && $compilerAvailable; then echo "Scala core already built!" else - $SBT_CMD --warn "setupPublishCore $prRepoUrl" publish + $SBT --warn "setupPublishCore $prRepoUrl" publish fi mv buildcharacter.properties jenkins.properties # parsed by the jenkins job diff --git a/scripts/jobs/validate/test b/scripts/jobs/validate/test index 3cd8af5608..39fafebaef 100755 --- a/scripts/jobs/validate/test +++ b/scripts/jobs/validate/test @@ -4,6 +4,9 @@ baseDir=${WORKSPACE-`pwd`} scriptsDir="$baseDir/scripts" . $scriptsDir/common +generateRepositoriesConfig $prRepoUrl +SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig" + case $prDryRun in yep) @@ -14,22 +17,13 @@ case $prDryRun in # build quick using STARR built upstream, as specified by scalaVersion # (in that sense it's locker, since it was built with starr by that upstream job); - # and run JUnit tests, partest, OSGi tests, MiMa and scaladoc - $SBT_CMD \ + # and run JUnit tests, ScalaCheck tests, partest, OSGi tests, MiMa and scaladoc + $SBT \ -Dstarr.version=$scalaVersion \ --warn \ "setupValidateTest $prRepoUrl" \ $testExtraArgs \ - "test" \ - "partest run pos neg jvm" \ - "partest res scalap specialized scalacheck" \ - "partest instrumented presentation" \ - "partest --srcpath scaladoc" \ - osgiTestFelix/test \ - osgiTestEclipse/test \ - library/mima \ - reflect/mima \ - doc + testAll ;; diff --git a/scripts/stability-test.sh b/scripts/stability-test.sh new file mode 100755 index 0000000000..f017ac0842 --- /dev/null +++ b/scripts/stability-test.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# + +declare failed + +echo "Comparing build/quick/classes and build/strap/classes" +for dir in library reflect compiler; do + # feel free to replace by a more elegant approach -- don't know how + if diff -rw -x '*.css' \ + -x '*.custom' \ + -x '*.gif' \ + -x '*.js' \ + -x '*.layout' \ + -x '*.png' \ + -x '*.properties' \ + -x '*.tmpl' \ + -x '*.tooltip' \ + -x '*.txt' \ + -x '*.xml' \ + build/{quick,strap}/classes/$dir + then + classes=$(find build/quick/classes/$dir -name '*.class' | wc -l) + printf "%8s: %5d classfiles verified identical\n" $dir $classes + else + failed=true + fi +done + +[[ -z $failed ]] || exit 127 |