From 42f813e18160b6ea8dfc9c2ae850a67dd0819773 Mon Sep 17 00:00:00 2001 From: Lukas Rytz Date: Fri, 3 Mar 2017 17:48:24 +0100 Subject: [backport] new repo, version numbers for integration builds Integration builds now have version number like `2.12.2-bin-sha7` or `2.13.0-pre-sha7` and are published to scala-integration (no longer scala-release-temp). scala-release-temp is still used in the bootstrap script for publishing locker. --- README.md | 2 +- project/VersionUtil.scala | 35 ++++-- scripts/jobs/integrate/bootstrap | 136 ++++++++++++--------- .../scala/tools/nsc/settings/ScalaVersion.scala | 2 +- 4 files changed, 103 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index bbfe7a797b..c2891390d4 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ tested version during CI validation. The Scala CI builds nightly download releases (including all modules) and publishes them to the following locations: - [2.12.x](http://www.scala-lang.org/files/archive/nightly/2.12.x/?C=M;O=D) - - [2.11.x](http://www.scala-lang.org/files/archive/nightly/2.11.x/?C=M;O=A) + - [2.11.x](http://www.scala-lang.org/files/archive/nightly/2.11.x/?C=M;O=D) The CI also publishes nightly API docs: - [2.12.x](http://www.scala-lang.org/files/archive/nightly/2.12.x/api/?C=M;O=D) diff --git a/project/VersionUtil.scala b/project/VersionUtil.scala index 2a52159d4e..3f9b727ef0 100644 --- a/project/VersionUtil.scala +++ b/project/VersionUtil.scala @@ -45,17 +45,19 @@ object VersionUtil { /** Compute the canonical, Maven and OSGi version number from `baseVersion` and `baseVersionSuffix`. * Examples of the generated versions: * - * ("2.11.8", "SNAPSHOT" ) -> ("2.11.8-20151215-133023-7559aed3c5", "2.11.8-SNAPSHOT", "2.11.8.v20151215-133023-7559aed3c5") - * ("2.11.8", "SHA-SNAPSHOT") -> ("2.11.8-20151215-133023-7559aed3c5", "2.11.8-7559aed3c5-SNAPSHOT", "2.11.8.v20151215-133023-7559aed3c5") - * ("2.11.8", "" ) -> ("2.11.8", "2.11.8", "2.11.8.v20151215-133023-VFINAL-7559aed3c5") - * ("2.11.8", "M3" ) -> ("2.11.8-M3", "2.11.8-M3", "2.11.8.v20151215-133023-M3-7559aed3c5") - * ("2.11.8", "RC4" ) -> ("2.11.8-RC4", "2.11.8-RC4", "2.11.8.v20151215-133023-RC4-7559aed3c5") - * ("2.11.8-RC4", "SPLIT" ) -> ("2.11.8-RC4", "2.11.8-RC4", "2.11.8.v20151215-133023-RC4-7559aed3c5") + * ("2.11.8", "SNAPSHOT" ) -> ("2.11.8-20151215-133023-7559aed", "2.11.8-bin-SNAPSHOT", "2.11.8.v20151215-133023-7559aed") + * ("2.11.8", "SHA-SNAPSHOT") -> ("2.11.8-20151215-133023-7559aed", "2.11.8-bin-7559aed-SNAPSHOT", "2.11.8.v20151215-133023-7559aed") + * ("2.11.8", "SHA" ) -> ("2.11.8-7559aed", "2.11.8-bin-7559aed", "2.11.8.v20151215-133023-7559aed") + * ("2.11.0", "SHA" ) -> ("2.11.0-7559aed", "2.11.0-pre-7559aed", "2.11.0.v20151215-133023-7559aed") + * ("2.11.8", "" ) -> ("2.11.8", "2.11.8", "2.11.8.v20151215-133023-VFINAL-7559aed") + * ("2.11.8", "M3" ) -> ("2.11.8-M3", "2.11.8-M3", "2.11.8.v20151215-133023-M3-7559aed") + * ("2.11.8", "RC4" ) -> ("2.11.8-RC4", "2.11.8-RC4", "2.11.8.v20151215-133023-RC4-7559aed") + * ("2.11.8-RC4", "SPLIT" ) -> ("2.11.8-RC4", "2.11.8-RC4", "2.11.8.v20151215-133023-RC4-7559aed") * * A `baseVersionSuffix` of "SNAPSHOT" is the default, which is used for local snapshot builds. The PR validation - * job uses "SHA-SNAPSHOT". An empty suffix is used for releases. All other suffix values are treated as RC / - * milestone builds. The special suffix value "SPLIT" is used to split the real suffix off from `baseVersion` - * instead and then apply the usual logic. */ + * job uses "SHA-SNAPSHOT". A proper version number for an integration build can be computed with "SHA". An empty + * suffix is used for releases. All other suffix values are treated as RC / milestone builds. The special suffix + * value "SPLIT" is used to split the real suffix off from `baseVersion` instead and then apply the usual logic. */ private lazy val versionPropertiesImpl: Def.Initialize[Versions] = Def.setting { val (base, suffix) = { @@ -78,11 +80,18 @@ object VersionUtil { val date = executeTool("get-scala-commit-date") val sha = executeTool("get-scala-commit-sha").substring(0, 7) // The script produces 10 digits at the moment + val Patch = """\d+\.\d+\.(\d+)""".r + def cross = base match { + case Patch(p) if p.toInt > 0 => "bin" + case _ => "pre" + } + val (canonicalV, mavenV, osgiV, release) = suffix match { - case "SNAPSHOT" => (s"$base-$date-$sha", s"$base-SNAPSHOT", s"$base.v$date-$sha", false) - case "SHA-SNAPSHOT" => (s"$base-$date-$sha", s"$base-$sha-SNAPSHOT", s"$base.v$date-$sha", false) - case "" => (s"$base", s"$base", s"$base.v$date-VFINAL-$sha", true) - case suffix => (s"$base-$suffix", s"$base-$suffix", s"$base.v$date-$suffix-$sha", true) + case "SNAPSHOT" => (s"$base-$date-$sha", s"$base-$cross-SNAPSHOT", s"$base.v$date-$sha", false) + case "SHA-SNAPSHOT" => (s"$base-$date-$sha", s"$base-$cross-$sha-SNAPSHOT", s"$base.v$date-$sha", false) + case "SHA" => (s"$base-$sha", s"$base-$cross-$sha", s"$base.v$date-$sha", false) + case "" => (s"$base", s"$base", s"$base.v$date-VFINAL-$sha", true) + case suffix => (s"$base-$suffix", s"$base-$suffix", s"$base.v$date-$suffix-$sha", true) } Versions(canonicalV, mavenV, osgiV, sha, date, release) diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap index abb5b283c6..216a93637b 100644 --- a/scripts/jobs/integrate/bootstrap +++ b/scripts/jobs/integrate/bootstrap @@ -3,16 +3,17 @@ # Script Overview # - determine scala version # - determine module versions -# - build minimal core (aka locker) of Scala, use the determined version number, publish to private-repo -# - build those modules where a binary compatible version doesn't exist, publish to private-repo -# - build Scala using the previously built core and bootstrap modules, publish to private-repo (overwrites the minimal core version on private-repo) -# - for releases (not nightlies) +# - build minimal core (aka locker) of Scala, use the determined version number, publish to scala-integration +# - build those modules where a binary compatible version doesn't exist, publish to scala-integration +# - build Scala using the previously built core and bootstrap modules, publish to scala-integration, overwriting +# the existing artifacts (note: in 2.12.x we use scala-release-temp for locker, but that doesn't work in 2.11.x, +# because we can only pass in one `extra.repo` to ant, see comment in PR 5764). +# - for releases # - stage Scala on sonatype # - rebuild modules that needed a rebuild with this Scala build, and stage them on sonatype # - for nightlies # - force rebuild all modules and publish them locally (for testing purposes) # - the Scala version is serialized to jenkins.properties, which is passed downstream to scala-release jobs -# - this removes the need to tag scala/scala-dist (it's still encouraged for releases, but not a hard requirement) # Specifying the Scala version: @@ -23,8 +24,8 @@ # - Note: After building a release, the jenkins job provides an updated versions.properties file as artifact. # Put this file in the Scala repo and create a pull request, and also update the file build.number. # -# - Otherwise, a nightly release is built: -# - version number is read from the build.number file, extended with -$sha-nightly +# - Otherwise, an integration build is performed: +# - version number is read from the build.number file, extended with -[bin|pre]-$sha # Specifying module versions: there are two modes @@ -56,7 +57,7 @@ # to be re-built using the 2.11.1 release, we could not use 2.11.0. We could also not release the modules # after 2.11.1 was out, because that way the scala-library-all pom of 2.11.1 would depend on the old modules. # -# (*) https://github.com/sbt/sbt/blob/0.13.8/util/cross/src/main/input_sources/CrossVersionUtil.scala#L39 +# (*) https://github.com/sbt/sbt/blob/v0.13.13/util/cross/src/main/input_sources/CrossVersionUtil.scala#L41 # Binary incompatible changes in Modules: example with Scala 2.11 / 2.12 and scala-parser-combinators @@ -107,24 +108,30 @@ mkdir -p $baseDir/ivy2 rm -rf $baseDir/resolutionScratch_ mkdir -p $baseDir/resolutionScratch_ -# repo used to publish "locker" scala to (to start the bootstrap) -releaseTempRepoCred="private-repo" -releaseTempRepoUrl=${releaseTempRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-release-temp/"} -jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} - -# Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs, -# so we need to configure SBT to use these rather than its default, Maven Central. -# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html -sbtRepositoryConfig="$scriptsDir/repositories-scala-release" -cat > "$sbtRepositoryConfig" << EOF +function generateRepositoriesConfig(){ + # Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs, + # so we need to configure SBT to use these rather than its default, Maven Central. + # See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html + sbtRepositoryConfig="$scriptsDir/repositories-scala-release" + jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"} + cat > "$sbtRepositoryConfig" << EOF [repositories] - private-repo: $releaseTempRepoUrl + script-repo: $1 jcenter-cache: $jcenterCacheUrl typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] maven-central local EOF +} + +integrationRepoCred="private-repo" + +# repo for locker, quick and the modules +integrationRepoUrl=${integrationRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-integration/"} + +# adding `integrationRepoUrl` to find the locker scala version when building modules +generateRepositoriesConfig $integrationRepoUrl ##### git gfxd() { @@ -198,15 +205,15 @@ sbtResolve() { # then set the version to the right one and publish (which won't re-gen the docs). # Also tried publish-local without docs using 'set publishArtifact in (Compile, packageDoc) := false' and republishing, no dice. -# Each buildModule() function is invoked twice: first to build against locker and publish to private-repo, then +# Each buildModule() function is invoked twice: first to build against locker and publish to artifactory, then # to build against the release and publish to sonatype (or publish-local if publishToSonatype is not "yes"). -# In the second round, sbtResolve is always true: the module will be found in the private-repo! +# In the second round, sbtResolve is always true: the module will be found in the artifactory! # Therefore, if MODULE_BUILT is "yes" (in the second round), we know that we need to build (and publish) the # module again. # -# Note: we tried an alternative solution in which sbtResolve would not look at private-repo, but that fails. For example, +# Note: we tried an alternative solution in which sbtResolve would not look at artifactory, but that fails. For example, # scala-xml depends on scala-library, so sbt tries to find the scala-library of the version that we are currently building, -# which exists only in private-repo. +# which exists only in artifactory. buildXML() { if [ "$XML_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-xml" $XML_VER ) @@ -281,7 +288,7 @@ buildActorsMigration(){ fi } -# should only be called with publishTasks publishing to private-repo +# should only be called with publishTasks publishing to artifactory buildScalacheck(){ if [ "$SCALACHECK_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scalacheck" "scalacheck" $SCALACHECK_VER ) then echo "Found scalacheck $SCALACHECK_VER; not building." @@ -292,9 +299,9 @@ buildScalacheck(){ fi } -# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to private-repo) +# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to artifactory) buildModules() { - publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"private-repo\" at \"$releaseTempRepoUrl\")") + publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")") buildTasks=($publishPrivateTask) buildXML buildParsers @@ -329,20 +336,19 @@ scalaVerToBinary() { local patch="$(echo $2 | sed -e "s#$RE#\3#")" # The binary version is majMin (e.g. "2.12") if - # - there's no suffix : 2.12.0, 2.12.1 - # - the suffix starts with "-bin": 2.12.0-bin-M1 - # - the patch version is > 0 : 2.12.1-M1, 1.12.3-RC2, 2.12.1-sha-nightly, 2.12.2-SNAPSHOT + # - there's no suffix : 2.12.0, 2.12.1 + # - the suffix starts with "-bin" : 2.12.1-bin-sha, 2.12.1-bin-sha-custom, 2.12.1-bin-SNAPSHOT + # - the suffix is \w+ and patch version is > 0: 2.12.1-M1, 2.12.1-RC2 (also 2.12.1-sha, 2.12.1-SNAPSHOT, which we don't use) # - # Otherwise, the binary version is the full version: 2.12.0-M1, 2.12.0-RC2, 2.12.0-sha-nightly, 2.12.0-SNAPSHOT + # Otherwise, the binary version is the full version: 2.12.0-M1, 2.12.0-RC2, 2.12.0-pre-sha, 2.12.0-pre-SNAPSHOT + # (also 2.12.0-sha, 2.12.0-SNAPSHOT, which we don't use) # - # Adapted from sbt: https://github.com/sbt/sbt/blob/0.13.8/util/cross/src/main/input_sources/CrossVersionUtil.scala#L39 + # Adapted from sbt: https://github.com/sbt/sbt/blob/v0.13.13/util/cross/src/main/input_sources/CrossVersionUtil.scala#L42 # - # Note: during the pre-release cycle of a major release (e.g. before 2.12.0), the SCALA_BINARY_VER of nightly / SNAPSHOT - # versions is the full version, e.g. 2.12.0-sha-nightly, so modules are always re-built. This is in line with what sbt - # does: for example, with scalaVersion := "2.12.0-SNAPSHOT", sbt will resolve scala-xml as scala-xml_2.12.0-SNAPSHOT. - # Once the 2.12.0 release is out, the binary version is 2.12 for all versions (e.g. for 2.12.1-sha-nightly). + # During the pre-release cycle of a major release (e.g. before 2.12.0), the SCALA_BINARY_VER of integration / SNAPSHOT + # versions is the full version, e.g. 2.12.0-pre-sha, so modules are always re-built. - if [[ "$3" == "" || "${3:0:4}" == "-bin" || "$patch" != "0" ]]; then + if [[ "$3" == "" || "${3:0:4}" == "-bin" || ("$patch" != "0" && "$3" =~ ^-[a-zA-Z0-9_]+$) ]]; then echo "$majMin" else echo "$1" @@ -363,10 +369,15 @@ determineScalaVersion() { if [ -z "$scalaTag" ] then - echo "No tag found, building nightly snapshot." + echo "No tag found, running an integration build." parseScalaProperties "build.number" SCALA_VER_BASE="$version_major.$version_minor.$version_patch" - SCALA_VER_SUFFIX="-$(git rev-parse --short HEAD)-nightly" + local shaSuffix=$(git rev-parse HEAD | cut -c1-7) + local cross="bin" + if [[ $SCALA_VER_BASE =~ ^.*\.0$ ]]; then + cross="pre" + fi + SCALA_VER_SUFFIX="-$cross-$shaSuffix" SCALADOC_SOURCE_LINKS_VER=$(git rev-parse HEAD) # TODO: publish nightly snapshot using this script - currently it's a separate jenkins job still running at EPFL. @@ -468,20 +479,31 @@ createNetrcFile() { grep 'password=' $1 | sed 's/password=\(.*\)/password \1/' >> $netrcFile } +# deletes existing artifacts (core and modules) matching the $SCALA_VER from the repository passed as argument removeExistingBuilds() { - createNetrcFile "$HOME/.credentials-private-repo" - local netrcFile="$HOME/.credentials-private-repo-netrc" - - local storageApiUrl=`echo $releaseTempRepoUrl | sed 's/\(scala-release-temp\)/api\/storage\/\1/'` - local scalaLangModules=`curl -s $storageApiUrl/org/scala-lang | jq -r '.children | .[] | "org/scala-lang" + .uri'` - - for module in "org/scalacheck" $scalaLangModules; do - local artifacts=`curl -s $storageApiUrl/$module | jq -r ".children | .[] | select(.uri | endswith(\"$SCALA_VER\")) | .uri"` - for artifact in $artifacts; do - echo "Deleting $releaseTempRepoUrl$module$artifact" - curl -s --netrc-file $netrcFile -X DELETE $releaseTempRepoUrl$module$artifact + local repoUrl=$1 + local repoPrefix="https://scala-ci.typesafe.com/artifactory/" + if [[ $repoUrl == "$repoPrefix"* ]]; then + local repoId=${1#$repoPrefix} + local storageApiUrl="${repoPrefix}api/storage/$repoId" + + createNetrcFile "$HOME/.credentials-private-repo" + local netrcFile="$HOME/.credentials-private-repo-netrc" + + # "module" is not a scala module (like scala-xml), but an artifact of a boostrap build. the variable + # contains: "org/scala-lang/modules", "org/scala-lang/scala-compiler", "org/scala-lang/scala-library", ... + local scalaLangModules=`curl -s $storageApiUrl/org/scala-lang | jq -r '.children | .[] | "org/scala-lang" + .uri' | grep -v actors-migration` + + for module in $scalaLangModules; do + local artifacts=`curl -s $storageApiUrl/$module | jq -r ".children | .[] | select(.uri | endswith(\"$SCALA_VER\")) | .uri"` + for artifact in $artifacts; do + echo "Deleting $repoUrl$module$artifact" + curl -s --netrc-file $netrcFile -X DELETE $repoUrl$module$artifact + done done - done + else + echo "Unknown repo, not deleting anything: $repoUrl" + fi } constructUpdatedModuleVersions() { @@ -507,7 +529,7 @@ constructUpdatedModuleVersions() { if [ ! -z "$SCALA_BINARY_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala.binary.version=$SCALA_BINARY_VER"); fi } -# build locker (scala + modules) and quick, publishing everything to private-repo +# build locker (scala + modules) and quick, publishing everything to artifactory bootstrap() { echo "### Bootstrapping" @@ -524,8 +546,8 @@ bootstrap() { # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler ant -Dmaven.version.number=$SCALA_VER\ -Dremote.snapshot.repository=NOPE\ - -Dremote.release.repository=$releaseTempRepoUrl\ - -Drepository.credentials.id=$releaseTempRepoCred\ + -Dremote.release.repository=$integrationRepoUrl\ + -Drepository.credentials.id=$integrationRepoCred\ -Dscalac.args.optimise=-optimise\ -Ddocs.skip=1\ -Dlocker.skip=1\ @@ -561,14 +583,14 @@ bootstrap() { # which is fully cross-versioned (for $SCALA_VER, the version we're releasing) ant -Dstarr.version=$SCALA_VER\ -Dscala.full.version=$SCALA_VER\ - -Dextra.repo.url=$releaseTempRepoUrl\ + -Dextra.repo.url=$integrationRepoUrl\ -Dmaven.version.suffix=$SCALA_VER_SUFFIX\ ${updatedModuleVersions[@]} \ -Dupdate.versions=1\ -Dscaladoc.git.commit=$SCALADOC_SOURCE_LINKS_VER\ -Dremote.snapshot.repository=NOPE\ - -Dremote.release.repository=$releaseTempRepoUrl\ - -Drepository.credentials.id=$releaseTempRepoCred\ + -Dremote.release.repository=$integrationRepoUrl\ + -Drepository.credentials.id=$integrationRepoCred\ -Dscalac.args.optimise=-optimise\ $antBuildTask $publishPrivateTask @@ -614,7 +636,7 @@ determineScalaVersion deriveModuleVersions -removeExistingBuilds +removeExistingBuilds $integrationRepoUrl bootstrap diff --git a/src/compiler/scala/tools/nsc/settings/ScalaVersion.scala b/src/compiler/scala/tools/nsc/settings/ScalaVersion.scala index 43bdad5882..d7901730a4 100644 --- a/src/compiler/scala/tools/nsc/settings/ScalaVersion.scala +++ b/src/compiler/scala/tools/nsc/settings/ScalaVersion.scala @@ -135,7 +135,7 @@ abstract class ScalaBuild extends Ordered[ScalaBuild] { def unparse: String } /** - * A development, test, nightly, snapshot or other "unofficial" build + * A development, test, integration, snapshot or other "unofficial" build */ case class Development(id: String) extends ScalaBuild { def unparse = s"-${id}" -- cgit v1.2.3