path: root/scripts/jobs/integrate/bootstrap

#!/bin/bash -e



# NOTE: this is a quick backport of the 2.11.x script to 2.10.x -- some comments may be out dated



# TODO: different scripts for the different phases -- usually we don't need to bootstrap the modules,
# since we can use the previous version of scala for STARR as well as for compiling the modules (assuming it's binary compatible)
# We should move away from the complicated bootstrap and set up our release schedule so we always have a previous build that satisfies these criteria.
# (Potentially trivially, by splitting up this script, and publishing locker as if it were a real release.)

# requirements:
# sbtCmd must point to sbt from sbt-extras (this is the standard on the Scala jenkins, so we only support that one)
# - ~/.sonatype-curl that consists of user = USER:PASS
# - ~/.m2/settings.xml with credentials for sonatype
    # <server>
    #   <id>private-repo</id>
    #   <username>jenkinside</username>
    #   <password></password>
    # </server>
# - ~/.credentials (for sonatype)
    # realm=Sonatype Nexus Repository Manager
    # host=oss.sonatype.org
    # user=lamp
    # password=
# - ~/.credentials-private-repo for private-repo.typesafe.com, as follows:
    # realm=Artifactory Realm
    # host=private-repo.typesafe.com
    # user=jenkinside
    # password=
# - ~/.sbt/0.13/plugins/gpg.sbt with:
#   addSbtPlugin("com.typesafe.sbt" % "sbt-pgp" % "0.8.1")

# Modus operandi:
#
# Determine Scala version as:
#
# $SCALA_VER_BASE$SCALA_VER_SUFFIX (if former variable is set)
# By parsing the tag (if HEAD is tagged as v$base$suffix)
# By parsing build.number for the base version, suffixing with -$sha-nightly
# Serialize these versions to jenkins.properties, which are passed downstream to scala-release-2.11.x-dist.
# This also removes the need to tag scala/scala-dist (not possible for nightlies, still encouraged for releases, but not a hard requirement).
#
# Determine Module Versions
#
# When running in "versions.properties" mode (the default), derive tags from these versions and build, publishing only those modules that are not available yet.
# Otherwise, build HEAD for all modules, derive a -nightly version for them.
# Bootstrap:
#
# Build minimal core of Scala as this version (aka locker), publish to private-repo
# Build modules required to bootstrap, publish to private-repo
# Build Scala using the previously built core and bootstrap modules, publish to private-repo This overwrites the minimal core on private-repo
# Stage to sonatype (unless building a -nightly release):
#
# Stage this Scala build on sonatype
# Rebuild modules with this Scala build, and stage them on sonatype as well
# This script can be run in multiple modes. It is design to work without any input,
# so that it could be run in Travis CI. In that mode, it'll build a release when
# the current HEAD of the checkout in $WORKSPACE is tagged, and stage to sonatype. Otherwise,
# it'll build a nightly.
#
# Since the nightlies are intended to be a drop in replacement, all modules are built with the
# full Scala version as their binary version, so that you can just set scalaVersion to the
# nightly's sha-derived version and be good to go.
#
# The other way to trigger a release is by setting the SCALA_VER_BASE env var.
#
# By default, we build the versions of the modules as specified by versions.properties
# (as specified in the HEAD commit). Set moduleVersioning to something random
# to trigger building HEAD of each module, generating a fresh -$(git describe)-nightly version for each.
#
# PS: set publishToSonatype to anything but "yes" to avoid publishing to sonatype
# (publishing only done when $WORKSPACE checkout's HEAD is tagged / SCALA_VER_BASE is set.)


sbtCmd=${sbtCmd-sbt} # TESTING (this is a marker for defaults to change when testing locally: should be sbtx on my mac)

# 0.13.5 does not respect "set every scalaVersion", see
# https://github.com/scala/scala-parser-combinators/pull/27
sbtCmd="$sbtCmd -sbt-version 0.13.2"

# publishToSonatype
# set to anything but "yes" to avoid publishing to sonatype
# overridden to "no" when no SCALA_VER_BASE is passed and HEAD is not tagged with a valid version tag
#

baseDir=${WORKSPACE-`pwd`}
scriptsDir="$baseDir/scripts"
. $scriptsDir/common

# we must change ivy home to get a fresh ivy cache, otherwise we get half-bootstrapped scala
# rm it in case it existed (and there's no ivy2-shadow, which indicates we're running in a TESTING environment)...
# we don't nuke the whole ws since that clobbers the git clones needlessly
[[ -d $baseDir/ivy2-shadow ]] || rm -rf $baseDir/ivy2
mkdir -p $baseDir/ivy2

rm -rf $baseDir/resolutionScratch_
mkdir -p $baseDir/resolutionScratch_

# repo used to publish "locker" scala to (to start the bootstrap)
privateCred="private-repo"
privateRepo="http://private-repo.typesafe.com/typesafe/scala-release-temp/"

##### git
gfxd() {
  git clean -fxd # TESTING
}

update() {
  [[ -d $baseDir ]] || mkdir -p $baseDir
  cd $baseDir

  if [ ! -d $baseDir/$2 ]; then git clone "https://github.com/$1/$2.git"; fi

  cd $2

  git fetch --tags "https://github.com/$1/$2.git"
  (git fetch "https://github.com/$1/$2.git" $3 && git checkout -q FETCH_HEAD) #|| git checkout -q $3 # || fallback is for local testing on tag
  git reset --hard
}

##### sonatype interface

stApi="https://oss.sonatype.org/service/local"

function st_curl(){
  curl -H "Content-Type: application/json" -H "accept: application/json,application/vnd.siesta-error-v1+json,application/vnd.siesta-validation-errors-v1+json"  -K ~/.sonatype-curl -s -o - $@
}

function st_stagingReposOpen() {
 st_curl "$stApi/staging/profile_repositories" | jq '.data[] | select(.profileName == "org.scala-lang") | select(.type == "open")'
}

function st_stagingRepoDrop() {
  repo=$1
  message=$2
  echo "{\"data\":{\"description\":\"$message\",\"stagedRepositoryIds\":[\"$repo\"]}}" | st_curl -X POST -d @- "$stApi/staging/bulk/drop"
}

function st_stagingRepoClose() {
  repo=$1
  message=$2
  echo "{\"data\":{\"description\":\"$message\",\"stagedRepositoryIds\":[\"$repo\"]}}" | st_curl -X POST -d @- "$stApi/staging/bulk/close"
}


# ARGH trying to get this to work on multiple versions of sbt-extras...
# the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir
# the new version of sbt-extras ignores sbt-dir, so we pass it in as -Dsbt.global.base
# need to set sbt-dir to one that has the gpg.sbt plugin config
sbtArgs="-no-colors -ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$scriptsDir/repositories-scala-release -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13"

sbtBuild() {
  echo "### sbtBuild: "$sbtCmd $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@"
  $sbtCmd $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1
}

sbtResolve() {
  cd $baseDir/resolutionScratch_
  touch build.sbt
  cross=${4-binary} # Disabled / binary / full
  echo "### sbtResolve: $sbtCmd $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
  $sbtCmd $sbtArgs "${scalaVersionTasks[@]}" \
    "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \
      'show update' >> $baseDir/logs/resolution 2>&1
}


## BUILD STEPS:

determineScalaVersion() {
  cd $WORKSPACE
  if [ -z "$SCALA_VER_BASE" ]; then
    echo "No SCALA_VER_BASE specified."

    scalaTag=$(git describe --exact-match ||:)

    SCALA_BINARY_VER=${SCALA_BINARY_VER-"$scala_binary_version"}

    if [ -z "$scalaTag" ]
    then
      echo "No tag found, building nightly snapshot."
      parseScalaProperties "build.number"
      SCALA_VER_BASE="$version_major.$version_minor.$version_patch"
      SCALA_VER_SUFFIX="-$(git rev-parse --short HEAD)-nightly"
      SCALADOC_SOURCE_LINKS_VER=$(git rev-parse HEAD)

      # TODO: publish nightly snapshot using this script
      publishToSonatype="no"
      echo "repo_ref=2.11.x" >> $baseDir/jenkins.properties # for the -dist downstream jobs that build the actual archives
    else
      echo "HEAD is tagged as $scalaTag."
      # borrowed from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
      local RE='v*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' # don't change this to make it more accurate, it's not worth it
      SCALA_VER_BASE="$(echo $scalaTag | sed -e "s#$RE#\1.\2.\3#")"
      SCALA_VER_SUFFIX="$(echo $scalaTag | sed -e "s#$RE#\4#")"
      SCALADOC_SOURCE_LINKS_VER=$scalaTag

      if [ "$SCALA_VER_BASE" == "$scalaTag" ]; then
        echo "Could not parse version $scalaTag"
        exit 1
      fi
      publishToSonatype=${publishToSonatype-"yes"} # unless forced previously, publish
    fi
  else
    publishToSonatype=${publishToSonatype-"yes"} # unless forced previously, publish
    # if version base/suffix are provided, we assume a corresponding tag exists for the scaladoc source links
    SCALADOC_SOURCE_LINKS_VER="v$SCALA_VER_BASE$SCALA_VER_SUFFIX"
  fi

  SCALA_VER="$SCALA_VER_BASE$SCALA_VER_SUFFIX"
  echo "version=$SCALA_VER" >> $baseDir/jenkins.properties
  echo "sbtDistVersionOverride=-Dproject.version=$SCALA_VER" >> $baseDir/jenkins.properties

  echo "Building Scala $SCALA_VER."
}

deriveVersion() {
  update $1 $2 $3 &> /dev/null
  echo "$(git describe --match=v* | cut -dv -f2)-nightly"
}

deriveVersionAnyTag() {
  update $1 $2 $3 &> /dev/null
  echo "$(git describe | cut -dv -f2)-nightly"
}

distro() {
  cd $WORKSPACE

  echo "### Building the distribution"

  ant -Dextra.repo.url=$privateRepo\
      -Dmaven.version.suffix=$SCALA_VER_SUFFIX\
      -Dscaladoc.git.commit=$SCALADOC_SOURCE_LINKS_VER\
      -Dremote.snapshot.repository=NOPE\
      -Dremote.release.repository=$privateRepo\
      -Drepository.credentials.id=$privateCred\
      distpack-opt

  (cd $WORKSPACE/dists/ && tar cvzhf ../scala-dist-$SCALA_VER.tar.gz .)
  s3Upload scala-dist-$SCALA_VER.tar.gz
}

s3Upload() {
  file="$1"
  contentType="application/x-compressed-tar"
  host=$(grep host ~/.s3credentials | cut -d= -f2)

  s3curl --id typesafe --contentType "${contentType}" --put "${file}" -- -k https://${host}/scala/tmp/${file}
}

# assumes we just bootstrapped, and current directory is $baseDir
# publishes locker to sonatype, then builds modules again (those for which version numbers were provided),
# and publishes those to sonatype as well
# finally, the staging repos are closed
publishSonatype() {
  # stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary,
  # since we're just publishing an existing build
  echo "### Publishing to sonatype"
  (cd $WORKSPACE/dists/maven/latest/ && ant deploy.signed)

  open=$(st_stagingReposOpen)
  allOpenUrls=$(echo $open | jq  '.repositoryURI' | tr -d \")
  allOpen=$(echo $open | jq  '.repositoryId' | tr -d \")

  echo "Closing open repos: $allOpen"

  for repo in $allOpen; do st_stagingRepoClose $repo; done

  echo "Closed sonatype staging repos: $allOpenUrls."
}


#### MAIN

determineScalaVersion

distro

if [ "$publishToSonatype" == "yes" ]
  then publishSonatype
fi