diff options
Diffstat (limited to 'scripts/jobs/integrate/bootstrap')
| -rw-r--r-- | scripts/jobs/integrate/bootstrap | 135 |
1 files changed, 78 insertions, 57 deletions
diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap index abb5b283c6..85be794276 100644 --- a/scripts/jobs/integrate/bootstrap +++ b/scripts/jobs/integrate/bootstrap @@ -3,16 +3,16 @@ # Script Overview # - determine scala version # - determine module versions -# - build minimal core (aka locker) of Scala, use the determined version number, publish to private-repo -# - build those modules where a binary compatible version doesn't exist, publish to private-repo -# - build Scala using the previously built core and bootstrap modules, publish to private-repo (overwrites the minimal core version on private-repo) -# - for releases (not nightlies) +# - build minimal core (aka locker) of Scala, use the determined version number, publish to scala-integration +# - build those modules where a binary compatible version doesn't exist, publish to scala-integration +# - build Scala using the previously built core and bootstrap modules, publish to scala-integration, overwriting +# the existing artifacts +# - for releases # - stage Scala on sonatype # - rebuild modules that needed a rebuild with this Scala build, and stage them on sonatype # - for nightlies # - force rebuild all modules and publish them locally (for testing purposes) # - the Scala version is serialized to jenkins.properties, which is passed downstream to scala-release jobs -# - this removes the need to tag scala/scala-dist (it's still encouraged for releases, but not a hard requirement) # Specifying the Scala version: @@ -23,8 +23,8 @@ # - Note: After building a release, the jenkins job provides an updated versions.properties file as artifact. # Put this file in the Scala repo and create a pull request, and also update the file build.number. # -# - Otherwise, a nightly release is built: -# - version number is read from the build.number file, extended with -$sha-nightly +# - Otherwise, an integration build is performed: +# - version number is read from the build.number file, extended with -[bin|pre]-$sha # Specifying module versions: there are two modes @@ -56,7 +56,7 @@ # to be re-built using the 2.11.1 release, we could not use 2.11.0. We could also not release the modules # after 2.11.1 was out, because that way the scala-library-all pom of 2.11.1 would depend on the old modules. # -# (*) https://github.com/sbt/sbt/blob/0.13.8/util/cross/src/main/input_sources/CrossVersionUtil.scala#L39 +# (*) https://github.com/sbt/sbt/blob/v0.13.13/util/cross/src/main/input_sources/CrossVersionUtil.scala#L41 # Binary incompatible changes in Modules: example with Scala 2.11 / 2.12 and scala-parser-combinators @@ -107,24 +107,30 @@ mkdir -p $baseDir/ivy2 rm -rf $baseDir/resolutionScratch_ mkdir -p $baseDir/resolutionScratch_ -# repo used to publish "locker" scala to (to start the bootstrap) -releaseTempRepoCred="private-repo" -releaseTempRepoUrl=${releaseTempRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-release-temp/"} -jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} - -# Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs, -# so we need to configure SBT to use these rather than its default, Maven Central. -# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html -sbtRepositoryConfig="$scriptsDir/repositories-scala-release" -cat > "$sbtRepositoryConfig" << EOF +function generateRepositoriesConfig(){ + # Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs, + # so we need to configure SBT to use these rather than its default, Maven Central. + # See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html + sbtRepositoryConfig="$scriptsDir/repositories-scala-release" + jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} + cat > "$sbtRepositoryConfig" << EOF [repositories] - private-repo: $releaseTempRepoUrl + script-repo: $1 jcenter-cache: $jcenterCacheUrl typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] maven-central local EOF +} + +integrationRepoCred="private-repo" + +# repo for locker, quick and the modules +integrationRepoUrl=${integrationRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-integration/"} + +# adding `integrationRepoUrl` to find the locker scala version when building modules +generateRepositoriesConfig $integrationRepoUrl ##### git gfxd() { @@ -198,15 +204,15 @@ sbtResolve() { # then set the version to the right one and publish (which won't re-gen the docs). # Also tried publish-local without docs using 'set publishArtifact in (Compile, packageDoc) := false' and republishing, no dice. -# Each buildModule() function is invoked twice: first to build against locker and publish to private-repo, then +# Each buildModule() function is invoked twice: first to build against locker and publish to artifactory, then # to build against the release and publish to sonatype (or publish-local if publishToSonatype is not "yes"). -# In the second round, sbtResolve is always true: the module will be found in the private-repo! +# In the second round, sbtResolve is always true: the module will be found in the artifactory! # Therefore, if MODULE_BUILT is "yes" (in the second round), we know that we need to build (and publish) the # module again. # -# Note: we tried an alternative solution in which sbtResolve would not look at private-repo, but that fails. For example, +# Note: we tried an alternative solution in which sbtResolve would not look at artifactory, but that fails. For example, # scala-xml depends on scala-library, so sbt tries to find the scala-library of the version that we are currently building, -# which exists only in private-repo. +# which exists only in artifactory. buildXML() { if [ "$XML_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-xml" $XML_VER ) @@ -281,7 +287,7 @@ buildActorsMigration(){ fi } -# should only be called with publishTasks publishing to private-repo +# should only be called with publishTasks publishing to artifactory buildScalacheck(){ if [ "$SCALACHECK_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scalacheck" "scalacheck" $SCALACHECK_VER ) then echo "Found scalacheck $SCALACHECK_VER; not building." @@ -292,9 +298,9 @@ buildScalacheck(){ fi } -# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to private-repo) +# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to artifactory) buildModules() { - publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"private-repo\" at \"$releaseTempRepoUrl\")") + publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")") buildTasks=($publishPrivateTask) buildXML buildParsers @@ -329,20 +335,19 @@ scalaVerToBinary() { local patch="$(echo $2 | sed -e "s#$RE#\3#")" # The binary version is majMin (e.g. "2.12") if - # - there's no suffix : 2.12.0, 2.12.1 - # - the suffix starts with "-bin": 2.12.0-bin-M1 - # - the patch version is > 0 : 2.12.1-M1, 1.12.3-RC2, 2.12.1-sha-nightly, 2.12.2-SNAPSHOT + # - there's no suffix : 2.12.0, 2.12.1 + # - the suffix starts with "-bin" : 2.12.1-bin-sha, 2.12.1-bin-sha-custom, 2.12.1-bin-SNAPSHOT + # - the suffix is \w+ and patch version is > 0: 2.12.1-M1, 2.12.1-RC2 (also 2.12.1-sha, 2.12.1-SNAPSHOT, which we don't use) # - # Otherwise, the binary version is the full version: 2.12.0-M1, 2.12.0-RC2, 2.12.0-sha-nightly, 2.12.0-SNAPSHOT + # Otherwise, the binary version is the full version: 2.12.0-M1, 2.12.0-RC2, 2.12.0-pre-sha, 2.12.0-pre-SNAPSHOT + # (also 2.12.0-sha, 2.12.0-SNAPSHOT, which we don't use) # - # Adapted from sbt: https://github.com/sbt/sbt/blob/0.13.8/util/cross/src/main/input_sources/CrossVersionUtil.scala#L39 + # Adapted from sbt: https://github.com/sbt/sbt/blob/v0.13.13/util/cross/src/main/input_sources/CrossVersionUtil.scala#L42 # - # Note: during the pre-release cycle of a major release (e.g. before 2.12.0), the SCALA_BINARY_VER of nightly / SNAPSHOT - # versions is the full version, e.g. 2.12.0-sha-nightly, so modules are always re-built. This is in line with what sbt - # does: for example, with scalaVersion := "2.12.0-SNAPSHOT", sbt will resolve scala-xml as scala-xml_2.12.0-SNAPSHOT. - # Once the 2.12.0 release is out, the binary version is 2.12 for all versions (e.g. for 2.12.1-sha-nightly). + # During the pre-release cycle of a major release (e.g. before 2.12.0), the SCALA_BINARY_VER of integration / SNAPSHOT + # versions is the full version, e.g. 2.12.0-pre-sha, so modules are always re-built. - if [[ "$3" == "" || "${3:0:4}" == "-bin" || "$patch" != "0" ]]; then + if [[ "$3" == "" || "${3:0:4}" == "-bin" || ("$patch" != "0" && "$3" =~ ^-[a-zA-Z0-9_]+$) ]]; then echo "$majMin" else echo "$1" @@ -363,10 +368,15 @@ determineScalaVersion() { if [ -z "$scalaTag" ] then - echo "No tag found, building nightly snapshot." + echo "No tag found, running an integration build." parseScalaProperties "build.number" SCALA_VER_BASE="$version_major.$version_minor.$version_patch" - SCALA_VER_SUFFIX="-$(git rev-parse --short HEAD)-nightly" + local shaSuffix=$(git rev-parse HEAD | cut -c1-7) + local cross="bin" + if [[ $SCALA_VER_BASE =~ ^.*\.0$ ]]; then + cross="pre" + fi + SCALA_VER_SUFFIX="-$cross-$shaSuffix" SCALADOC_SOURCE_LINKS_VER=$(git rev-parse HEAD) # TODO: publish nightly snapshot using this script - currently it's a separate jenkins job still running at EPFL. @@ -468,20 +478,31 @@ createNetrcFile() { grep 'password=' $1 | sed 's/password=\(.*\)/password \1/' >> $netrcFile } +# deletes existing artifacts (core and modules) matching the $SCALA_VER from the repository passed as argument removeExistingBuilds() { - createNetrcFile "$HOME/.credentials-private-repo" - local netrcFile="$HOME/.credentials-private-repo-netrc" - - local storageApiUrl=`echo $releaseTempRepoUrl | sed 's/\(scala-release-temp\)/api\/storage\/\1/'` - local scalaLangModules=`curl -s $storageApiUrl/org/scala-lang | jq -r '.children | .[] | "org/scala-lang" + .uri'` - - for module in "org/scalacheck" $scalaLangModules; do - local artifacts=`curl -s $storageApiUrl/$module | jq -r ".children | .[] | select(.uri | endswith(\"$SCALA_VER\")) | .uri"` - for artifact in $artifacts; do - echo "Deleting $releaseTempRepoUrl$module$artifact" - curl -s --netrc-file $netrcFile -X DELETE $releaseTempRepoUrl$module$artifact + local repoUrl=$1 + local repoPrefix="https://scala-ci.typesafe.com/artifactory/" + if [[ $repoUrl == "$repoPrefix"* ]]; then + local repoId=${1#$repoPrefix} + local storageApiUrl="${repoPrefix}api/storage/$repoId" + + createNetrcFile "$HOME/.credentials-private-repo" + local netrcFile="$HOME/.credentials-private-repo-netrc" + + # "module" is not a scala module (like scala-xml), but an artifact of a boostrap build. the variable + # contains: "org/scala-lang/modules", "org/scala-lang/scala-compiler", "org/scala-lang/scala-library", ... + local scalaLangModules=`curl -s $storageApiUrl/org/scala-lang | jq -r '.children | .[] | "org/scala-lang" + .uri' | grep -v actors-migration` + + for module in $scalaLangModules; do + local artifacts=`curl -s $storageApiUrl/$module | jq -r ".children | .[] | select(.uri | endswith(\"$SCALA_VER\")) | .uri"` + for artifact in $artifacts; do + echo "Deleting $repoUrl$module$artifact" + curl -s --netrc-file $netrcFile -X DELETE $repoUrl$module$artifact + done done - done + else + echo "Unknown repo, not deleting anything: $repoUrl" + fi } constructUpdatedModuleVersions() { @@ -507,7 +528,7 @@ constructUpdatedModuleVersions() { if [ ! -z "$SCALA_BINARY_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala.binary.version=$SCALA_BINARY_VER"); fi } -# build locker (scala + modules) and quick, publishing everything to private-repo +# build locker (scala + modules) and quick, publishing everything to artifactory bootstrap() { echo "### Bootstrapping" @@ -524,8 +545,8 @@ bootstrap() { # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler ant -Dmaven.version.number=$SCALA_VER\ -Dremote.snapshot.repository=NOPE\ - -Dremote.release.repository=$releaseTempRepoUrl\ - -Drepository.credentials.id=$releaseTempRepoCred\ + -Dremote.release.repository=$integrationRepoUrl\ + -Drepository.credentials.id=$integrationRepoCred\ -Dscalac.args.optimise=-optimise\ -Ddocs.skip=1\ -Dlocker.skip=1\ @@ -561,14 +582,14 @@ bootstrap() { # which is fully cross-versioned (for $SCALA_VER, the version we're releasing) ant -Dstarr.version=$SCALA_VER\ -Dscala.full.version=$SCALA_VER\ - -Dextra.repo.url=$releaseTempRepoUrl\ + -Dextra.repo.url=$integrationRepoUrl\ -Dmaven.version.suffix=$SCALA_VER_SUFFIX\ ${updatedModuleVersions[@]} \ -Dupdate.versions=1\ -Dscaladoc.git.commit=$SCALADOC_SOURCE_LINKS_VER\ -Dremote.snapshot.repository=NOPE\ - -Dremote.release.repository=$releaseTempRepoUrl\ - -Drepository.credentials.id=$releaseTempRepoCred\ + -Dremote.release.repository=$integrationRepoUrl\ + -Drepository.credentials.id=$integrationRepoCred\ -Dscalac.args.optimise=-optimise\ $antBuildTask $publishPrivateTask @@ -614,7 +635,7 @@ determineScalaVersion deriveModuleVersions -removeExistingBuilds +removeExistingBuilds $integrationRepoUrl bootstrap |