Merge pull request #5764 from lrytz/bootstrap-2.11

[backport] new repo, version numbers for integration builds

4 files changed, 103 insertions, 72 deletions

diff --git a/README.md b/README.md
index bbfe7a797b..c2891390d4 100644
--- a/README.md
+++ b/README.md
@@ -230,7 +230,7 @@ tested version during CI validation.
 The Scala CI builds nightly download releases (including all modules) and publishes
 them to the following locations:
   - [2.12.x](http://www.scala-lang.org/files/archive/nightly/2.12.x/?C=M;O=D)
-  - [2.11.x](http://www.scala-lang.org/files/archive/nightly/2.11.x/?C=M;O=A)
+  - [2.11.x](http://www.scala-lang.org/files/archive/nightly/2.11.x/?C=M;O=D)
 
 The CI also publishes nightly API docs:
   - [2.12.x](http://www.scala-lang.org/files/archive/nightly/2.12.x/api/?C=M;O=D)
diff --git a/project/VersionUtil.scala b/project/VersionUtil.scala
index 2a52159d4e..3f9b727ef0 100644
--- a/project/VersionUtil.scala
+++ b/project/VersionUtil.scala
@@ -45,17 +45,19 @@ object VersionUtil {
   /** Compute the canonical, Maven and OSGi version number from `baseVersion` and `baseVersionSuffix`.
     * Examples of the generated versions:
     *
-    * ("2.11.8", "SNAPSHOT"    ) -> ("2.11.8-20151215-133023-7559aed3c5", "2.11.8-SNAPSHOT",            "2.11.8.v20151215-133023-7559aed3c5")
-    * ("2.11.8", "SHA-SNAPSHOT") -> ("2.11.8-20151215-133023-7559aed3c5", "2.11.8-7559aed3c5-SNAPSHOT", "2.11.8.v20151215-133023-7559aed3c5")
-    * ("2.11.8", ""            ) -> ("2.11.8",                            "2.11.8",                     "2.11.8.v20151215-133023-VFINAL-7559aed3c5")
-    * ("2.11.8", "M3"          ) -> ("2.11.8-M3",                         "2.11.8-M3",                  "2.11.8.v20151215-133023-M3-7559aed3c5")
-    * ("2.11.8", "RC4"         ) -> ("2.11.8-RC4",                        "2.11.8-RC4",                 "2.11.8.v20151215-133023-RC4-7559aed3c5")
-    * ("2.11.8-RC4", "SPLIT"   ) -> ("2.11.8-RC4",                        "2.11.8-RC4",                 "2.11.8.v20151215-133023-RC4-7559aed3c5")
+    * ("2.11.8", "SNAPSHOT"    ) -> ("2.11.8-20151215-133023-7559aed", "2.11.8-bin-SNAPSHOT",         "2.11.8.v20151215-133023-7559aed")
+    * ("2.11.8", "SHA-SNAPSHOT") -> ("2.11.8-20151215-133023-7559aed", "2.11.8-bin-7559aed-SNAPSHOT", "2.11.8.v20151215-133023-7559aed")
+    * ("2.11.8", "SHA"         ) -> ("2.11.8-7559aed",                 "2.11.8-bin-7559aed",          "2.11.8.v20151215-133023-7559aed")
+    * ("2.11.0", "SHA"         ) -> ("2.11.0-7559aed",                 "2.11.0-pre-7559aed",          "2.11.0.v20151215-133023-7559aed")
+    * ("2.11.8", ""            ) -> ("2.11.8",                         "2.11.8",                      "2.11.8.v20151215-133023-VFINAL-7559aed")
+    * ("2.11.8", "M3"          ) -> ("2.11.8-M3",                      "2.11.8-M3",                   "2.11.8.v20151215-133023-M3-7559aed")
+    * ("2.11.8", "RC4"         ) -> ("2.11.8-RC4",                     "2.11.8-RC4",                  "2.11.8.v20151215-133023-RC4-7559aed")
+    * ("2.11.8-RC4", "SPLIT"   ) -> ("2.11.8-RC4",                     "2.11.8-RC4",                  "2.11.8.v20151215-133023-RC4-7559aed")
     *
     * A `baseVersionSuffix` of "SNAPSHOT" is the default, which is used for local snapshot builds. The PR validation
-    * job uses "SHA-SNAPSHOT". An empty suffix is used for releases. All other suffix values are treated as RC /
-    * milestone builds. The special suffix value "SPLIT" is used to split the real suffix off from `baseVersion`
-    * instead and then apply the usual logic. */
+    * job uses "SHA-SNAPSHOT". A proper version number for an integration build can be computed with "SHA". An empty
+    * suffix is used for releases. All other suffix values are treated as RC / milestone builds. The special suffix
+    * value "SPLIT" is used to split the real suffix off from `baseVersion` instead and then apply the usual logic. */
   private lazy val versionPropertiesImpl: Def.Initialize[Versions] = Def.setting {
 
     val (base, suffix) = {
@@ -78,11 +80,18 @@ object VersionUtil {
     val date = executeTool("get-scala-commit-date")
     val sha = executeTool("get-scala-commit-sha").substring(0, 7) // The script produces 10 digits at the moment
 
+    val Patch = """\d+\.\d+\.(\d+)""".r
+    def cross = base match {
+      case Patch(p) if p.toInt > 0 => "bin"
+      case _ => "pre"
+    }
+
     val (canonicalV, mavenV, osgiV, release) = suffix match {
-      case "SNAPSHOT"     => (s"$base-$date-$sha", s"$base-SNAPSHOT",      s"$base.v$date-$sha",         false)
-      case "SHA-SNAPSHOT" => (s"$base-$date-$sha", s"$base-$sha-SNAPSHOT", s"$base.v$date-$sha",         false)
-      case ""             => (s"$base",            s"$base",               s"$base.v$date-VFINAL-$sha",  true)
-      case suffix         => (s"$base-$suffix",    s"$base-$suffix",       s"$base.v$date-$suffix-$sha", true)
+      case "SNAPSHOT"     => (s"$base-$date-$sha", s"$base-$cross-SNAPSHOT",      s"$base.v$date-$sha",         false)
+      case "SHA-SNAPSHOT" => (s"$base-$date-$sha", s"$base-$cross-$sha-SNAPSHOT", s"$base.v$date-$sha",         false)
+      case "SHA"          => (s"$base-$sha",       s"$base-$cross-$sha",          s"$base.v$date-$sha",         false)
+      case ""             => (s"$base",            s"$base",                      s"$base.v$date-VFINAL-$sha",  true)
+      case suffix         => (s"$base-$suffix",    s"$base-$suffix",              s"$base.v$date-$suffix-$sha", true)
     }
 
     Versions(canonicalV, mavenV, osgiV, sha, date, release)
diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap
index abb5b283c6..216a93637b 100644
--- a/scripts/jobs/integrate/bootstrap
+++ b/scripts/jobs/integrate/bootstrap
@@ -3,16 +3,17 @@
 # Script Overview
 #  - determine scala version
 #  - determine module versions
-#  - build minimal core (aka locker) of Scala, use the determined version number, publish to private-repo
-#  - build those modules where a binary compatible version doesn't exist, publish to private-repo
-#  - build Scala using the previously built core and bootstrap modules, publish to private-repo (overwrites the minimal core version on private-repo)
-#  - for releases (not nightlies)
+#  - build minimal core (aka locker) of Scala, use the determined version number, publish to scala-integration
+#  - build those modules where a binary compatible version doesn't exist, publish to scala-integration
+#  - build Scala using the previously built core and bootstrap modules, publish to scala-integration, overwriting
+#    the existing artifacts (note: in 2.12.x we use scala-release-temp for locker, but that doesn't work in 2.11.x,
+#    because we can only pass in one `extra.repo` to ant, see comment in PR 5764).
+#  - for releases
 #    - stage Scala on sonatype
 #    - rebuild modules that needed a rebuild with this Scala build, and stage them on sonatype
 #  - for nightlies
 #    - force rebuild all modules and publish them locally (for testing purposes)
 #  - the Scala version is serialized to jenkins.properties, which is passed downstream to scala-release jobs
-#    - this removes the need to tag scala/scala-dist (it's still encouraged for releases, but not a hard requirement)
 
 
 # Specifying the Scala version:
@@ -23,8 +24,8 @@
 #    - Note: After building a release, the jenkins job provides an updated versions.properties file as artifact.
 #      Put this file in the Scala repo and create a pull request, and also update the file build.number.
 #
-# - Otherwise, a nightly release is built:
-#    - version number is read from the build.number file, extended with -$sha-nightly
+# - Otherwise, an integration build is performed:
+#    - version number is read from the build.number file, extended with -[bin|pre]-$sha
 
 
 # Specifying module versions: there are two modes
@@ -56,7 +57,7 @@
 #            to be re-built using the 2.11.1 release, we could not use 2.11.0. We could also not release the modules
 #            after 2.11.1 was out, because that way the scala-library-all pom of 2.11.1 would depend on the old modules.
 #
-# (*) https://github.com/sbt/sbt/blob/0.13.8/util/cross/src/main/input_sources/CrossVersionUtil.scala#L39
+# (*) https://github.com/sbt/sbt/blob/v0.13.13/util/cross/src/main/input_sources/CrossVersionUtil.scala#L41
 
 
 # Binary incompatible changes in Modules: example with Scala 2.11 / 2.12 and scala-parser-combinators
@@ -107,24 +108,30 @@ mkdir -p $baseDir/ivy2
 rm -rf $baseDir/resolutionScratch_
 mkdir -p $baseDir/resolutionScratch_
 
-# repo used to publish "locker" scala to (to start the bootstrap)
-releaseTempRepoCred="private-repo"
-releaseTempRepoUrl=${releaseTempRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-release-temp/"}
-jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
-
-# Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs,
-# so we need to configure SBT to use these rather than its default, Maven Central.
-# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html
-sbtRepositoryConfig="$scriptsDir/repositories-scala-release"
-cat > "$sbtRepositoryConfig" << EOF
+function generateRepositoriesConfig(){
+    # Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs,
+    # so we need to configure SBT to use these rather than its default, Maven Central.
+    # See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html
+    sbtRepositoryConfig="$scriptsDir/repositories-scala-release"
+    jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
+    cat > "$sbtRepositoryConfig" << EOF
 [repositories]
-  private-repo: $releaseTempRepoUrl
+  script-repo: $1
   jcenter-cache: $jcenterCacheUrl
   typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
   sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
   maven-central
   local
 EOF
+}
+
+integrationRepoCred="private-repo"
+
+# repo for locker, quick and the modules
+integrationRepoUrl=${integrationRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-integration/"}
+
+# adding `integrationRepoUrl` to find the locker scala version when building modules
+generateRepositoriesConfig $integrationRepoUrl
 
 ##### git
 gfxd() {
@@ -198,15 +205,15 @@ sbtResolve() {
 # then set the version to the right one and publish (which won't re-gen the docs).
 # Also tried publish-local without docs using 'set publishArtifact in (Compile, packageDoc) := false' and republishing, no dice.
 
-# Each buildModule() function is invoked twice: first to build against locker and publish to private-repo, then
+# Each buildModule() function is invoked twice: first to build against locker and publish to artifactory, then
 # to build against the release and publish to sonatype (or publish-local if publishToSonatype is not "yes").
-# In the second round, sbtResolve is always true: the module will be found in the private-repo!
+# In the second round, sbtResolve is always true: the module will be found in the artifactory!
 # Therefore, if MODULE_BUILT is "yes" (in the second round), we know that we need to build (and publish) the
 # module again.
 #
-# Note: we tried an alternative solution in which sbtResolve would not look at private-repo, but that fails. For example,
+# Note: we tried an alternative solution in which sbtResolve would not look at artifactory, but that fails. For example,
 # scala-xml depends on scala-library, so sbt tries to find the scala-library of the version that we are currently building,
-# which exists only in private-repo.
+# which exists only in artifactory.
 
 buildXML() {
   if [ "$XML_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules"  "scala-xml" $XML_VER )
@@ -281,7 +288,7 @@ buildActorsMigration(){
   fi
 }
 
-# should only be called with publishTasks publishing to private-repo
+# should only be called with publishTasks publishing to artifactory
 buildScalacheck(){
   if [ "$SCALACHECK_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scalacheck"  "scalacheck" $SCALACHECK_VER )
   then echo "Found scalacheck $SCALACHECK_VER; not building."
@@ -292,9 +299,9 @@ buildScalacheck(){
   fi
 }
 
-# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to private-repo)
+# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to artifactory)
 buildModules() {
-  publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"private-repo\" at \"$releaseTempRepoUrl\")")
+  publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")")
   buildTasks=($publishPrivateTask)
   buildXML
   buildParsers
@@ -329,20 +336,19 @@ scalaVerToBinary() {
   local patch="$(echo $2 | sed -e "s#$RE#\3#")"
 
   # The binary version is majMin (e.g. "2.12") if
-  #  - there's no suffix            : 2.12.0, 2.12.1
-  #  - the suffix starts with "-bin": 2.12.0-bin-M1
-  #  - the patch version is > 0     : 2.12.1-M1, 1.12.3-RC2, 2.12.1-sha-nightly, 2.12.2-SNAPSHOT
+  #  - there's no suffix                         : 2.12.0, 2.12.1
+  #  - the suffix starts with "-bin"             : 2.12.1-bin-sha, 2.12.1-bin-sha-custom, 2.12.1-bin-SNAPSHOT
+  #  - the suffix is \w+ and patch version is > 0: 2.12.1-M1, 2.12.1-RC2 (also 2.12.1-sha, 2.12.1-SNAPSHOT, which we don't use)
   #
-  # Otherwise, the binary version is the full version: 2.12.0-M1, 2.12.0-RC2, 2.12.0-sha-nightly, 2.12.0-SNAPSHOT
+  # Otherwise, the binary version is the full version: 2.12.0-M1, 2.12.0-RC2, 2.12.0-pre-sha, 2.12.0-pre-SNAPSHOT
+  # (also 2.12.0-sha, 2.12.0-SNAPSHOT, which we don't use)
   #
-  # Adapted from sbt: https://github.com/sbt/sbt/blob/0.13.8/util/cross/src/main/input_sources/CrossVersionUtil.scala#L39
+  # Adapted from sbt: https://github.com/sbt/sbt/blob/v0.13.13/util/cross/src/main/input_sources/CrossVersionUtil.scala#L42
   #
-  # Note: during the pre-release cycle of a major release (e.g. before 2.12.0), the SCALA_BINARY_VER of nightly / SNAPSHOT
-  # versions is the full version, e.g. 2.12.0-sha-nightly, so modules are always re-built. This is in line with what sbt
-  # does: for example, with scalaVersion := "2.12.0-SNAPSHOT", sbt will resolve scala-xml as scala-xml_2.12.0-SNAPSHOT.
-  # Once the 2.12.0 release is out, the binary version is 2.12 for all versions (e.g. for 2.12.1-sha-nightly).
+  # During the pre-release cycle of a major release (e.g. before 2.12.0), the SCALA_BINARY_VER of integration / SNAPSHOT
+  # versions is the full version, e.g. 2.12.0-pre-sha, so modules are always re-built.
 
-  if [[ "$3" == "" || "${3:0:4}" == "-bin" || "$patch" != "0" ]]; then
+  if [[ "$3" == "" || "${3:0:4}" == "-bin" || ("$patch" != "0" && "$3" =~ ^-[a-zA-Z0-9_]+$) ]]; then
     echo "$majMin"
   else
     echo "$1"
@@ -363,10 +369,15 @@ determineScalaVersion() {
 
     if [ -z "$scalaTag" ]
     then
-      echo "No tag found, building nightly snapshot."
+      echo "No tag found, running an integration build."
       parseScalaProperties "build.number"
       SCALA_VER_BASE="$version_major.$version_minor.$version_patch"
-      SCALA_VER_SUFFIX="-$(git rev-parse --short HEAD)-nightly"
+      local shaSuffix=$(git rev-parse HEAD | cut -c1-7)
+      local cross="bin"
+      if [[ $SCALA_VER_BASE =~ ^.*\.0$ ]]; then
+        cross="pre"
+      fi
+      SCALA_VER_SUFFIX="-$cross-$shaSuffix"
       SCALADOC_SOURCE_LINKS_VER=$(git rev-parse HEAD)
 
       # TODO: publish nightly snapshot using this script - currently it's a separate jenkins job still running at EPFL.
@@ -468,20 +479,31 @@ createNetrcFile() {
   grep 'password=' $1 | sed 's/password=\(.*\)/password \1/' >> $netrcFile
 }
 
+# deletes existing artifacts (core and modules) matching the $SCALA_VER from the repository passed as argument
 removeExistingBuilds() {
-  createNetrcFile "$HOME/.credentials-private-repo"
-  local netrcFile="$HOME/.credentials-private-repo-netrc"
-
-  local storageApiUrl=`echo $releaseTempRepoUrl | sed 's/\(scala-release-temp\)/api\/storage\/\1/'`
-  local scalaLangModules=`curl -s $storageApiUrl/org/scala-lang | jq -r '.children | .[] | "org/scala-lang" + .uri'`
-
-  for module in "org/scalacheck" $scalaLangModules; do
-    local artifacts=`curl -s $storageApiUrl/$module | jq -r ".children | .[] | select(.uri | endswith(\"$SCALA_VER\")) | .uri"`
-    for artifact in $artifacts; do
-      echo "Deleting $releaseTempRepoUrl$module$artifact"
-      curl -s --netrc-file $netrcFile -X DELETE $releaseTempRepoUrl$module$artifact
+  local repoUrl=$1
+  local repoPrefix="https://scala-ci.typesafe.com/artifactory/"
+  if [[ $repoUrl == "$repoPrefix"* ]]; then
+    local repoId=${1#$repoPrefix}
+    local storageApiUrl="${repoPrefix}api/storage/$repoId"
+
+    createNetrcFile "$HOME/.credentials-private-repo"
+    local netrcFile="$HOME/.credentials-private-repo-netrc"
+
+    # "module" is not a scala module (like scala-xml), but an artifact of a boostrap build. the variable
+    # contains: "org/scala-lang/modules", "org/scala-lang/scala-compiler", "org/scala-lang/scala-library", ...
+    local scalaLangModules=`curl -s $storageApiUrl/org/scala-lang | jq -r '.children | .[] | "org/scala-lang" + .uri' | grep -v actors-migration`
+
+    for module in $scalaLangModules; do
+      local artifacts=`curl -s $storageApiUrl/$module | jq -r ".children | .[] | select(.uri | endswith(\"$SCALA_VER\")) | .uri"`
+      for artifact in $artifacts; do
+        echo "Deleting $repoUrl$module$artifact"
+        curl -s --netrc-file $netrcFile -X DELETE $repoUrl$module$artifact
+      done
     done
-  done
+  else
+    echo "Unknown repo, not deleting anything: $repoUrl"
+  fi
 }
 
 constructUpdatedModuleVersions() {
@@ -507,7 +529,7 @@ constructUpdatedModuleVersions() {
   if [ ! -z "$SCALA_BINARY_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala.binary.version=$SCALA_BINARY_VER"); fi
 }
 
-# build locker (scala + modules) and quick, publishing everything to private-repo
+# build locker (scala + modules) and quick, publishing everything to artifactory
 bootstrap() {
   echo "### Bootstrapping"
 
@@ -524,8 +546,8 @@ bootstrap() {
   # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler
   ant -Dmaven.version.number=$SCALA_VER\
       -Dremote.snapshot.repository=NOPE\
-      -Dremote.release.repository=$releaseTempRepoUrl\
-      -Drepository.credentials.id=$releaseTempRepoCred\
+      -Dremote.release.repository=$integrationRepoUrl\
+      -Drepository.credentials.id=$integrationRepoCred\
       -Dscalac.args.optimise=-optimise\
       -Ddocs.skip=1\
       -Dlocker.skip=1\
@@ -561,14 +583,14 @@ bootstrap() {
   # which is fully cross-versioned (for $SCALA_VER, the version we're releasing)
   ant -Dstarr.version=$SCALA_VER\
       -Dscala.full.version=$SCALA_VER\
-      -Dextra.repo.url=$releaseTempRepoUrl\
+      -Dextra.repo.url=$integrationRepoUrl\
       -Dmaven.version.suffix=$SCALA_VER_SUFFIX\
       ${updatedModuleVersions[@]} \
       -Dupdate.versions=1\
       -Dscaladoc.git.commit=$SCALADOC_SOURCE_LINKS_VER\
       -Dremote.snapshot.repository=NOPE\
-      -Dremote.release.repository=$releaseTempRepoUrl\
-      -Drepository.credentials.id=$releaseTempRepoCred\
+      -Dremote.release.repository=$integrationRepoUrl\
+      -Drepository.credentials.id=$integrationRepoCred\
       -Dscalac.args.optimise=-optimise\
       $antBuildTask $publishPrivateTask
 
@@ -614,7 +636,7 @@ determineScalaVersion
 
 deriveModuleVersions
 
-removeExistingBuilds
+removeExistingBuilds $integrationRepoUrl
 
 bootstrap
 
diff --git a/src/compiler/scala/tools/nsc/settings/ScalaVersion.scala b/src/compiler/scala/tools/nsc/settings/ScalaVersion.scala
index 43bdad5882..d7901730a4 100644
--- a/src/compiler/scala/tools/nsc/settings/ScalaVersion.scala
+++ b/src/compiler/scala/tools/nsc/settings/ScalaVersion.scala
@@ -135,7 +135,7 @@ abstract class ScalaBuild extends Ordered[ScalaBuild] {
   def unparse: String
 }
 /**
- * A development, test, nightly, snapshot or other "unofficial" build
+ * A development, test, integration, snapshot or other "unofficial" build
  */
 case class Development(id: String) extends ScalaBuild {
   def unparse = s"-${id}"