diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | build.sbt | 32 | ||||
| -rw-r--r-- | project/ScriptCommands.scala | 19 | ||||
| -rw-r--r-- | project/VersionUtil.scala | 99 | ||||
| -rw-r--r-- | scripts/common | 3 | ||||
| -rwxr-xr-x | scripts/jobs/integrate/bootstrap | 20 | ||||
| -rwxr-xr-x | scripts/jobs/validate/publish-core | 13 |
7 files changed, 121 insertions, 68 deletions
diff --git a/.gitignore b/.gitignore index d6571a377f..061cd274ac 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ /build.properties /buildcharacter.properties +# might get generated when testing Jenkins scripts locally +/jenkins.properties + # target directories for ant build /build/ /dists/ @@ -53,7 +53,7 @@ * https://groups.google.com/d/topic/scala-internals/gp5JsM1E0Fo/discussion */ -import VersionUtil.{versionProps, versionNumber, generatePropertiesFileSettings, versionProperties, versionPropertiesSettings} +import VersionUtil._ val bootstrapScalaVersion = versionProps("starr.version") @@ -98,13 +98,27 @@ lazy val publishSettings : Seq[Setting[_]] = Seq( (f, to) } IO.copy(mappings) - } + }, + credentials ++= { + val file = Path.userHome / ".credentials" + if (file.exists) List(Credentials(file)) + else Nil + }, + publishMavenStyle := true ) -lazy val commonSettings = clearSourceAndResourceDirectories ++ versionPropertiesSettings ++ publishSettings ++ Seq[Setting[_]]( +// Set the version number: The ANT build uses the file "build.number" to get the base version. Overriding versions or +// suffixes for certain builds is done by directly setting variables from the shell scripts. For example, in +// publish-core this requires computing the commit SHA first and then passing it to ANT. In the sbt build we use +// the two settings `baseVersion` and `baseVersionSuffix` to compute all versions (canonical, Maven, OSGi). See +// VersionUtil.versionPropertiesImpl for details. The standard sbt `version` setting should not be set directly. It +// is the same as the Maven version and derived automatically from `baseVersion` and `baseVersionSuffix`. +globalVersionSettings +baseVersion in Global := "2.11.8" +baseVersionSuffix in Global := "SNAPSHOT" + +lazy val commonSettings = clearSourceAndResourceDirectories ++ publishSettings ++ Seq[Setting[_]]( organization := "org.scala-lang", - // The ANT build uses the file "build.number" and the property "build.release" to compute the version - version := "2.11.8-SNAPSHOT", scalaVersion := bootstrapScalaVersion, // we don't cross build Scala itself crossPaths := false, @@ -649,7 +663,13 @@ lazy val scalaDist = Project("scala-dist", file(".") / "target" / "scala-dist-di lazy val root = (project in file(".")) .settings(disableDocs: _*) - .settings(publishArtifact := false) + .settings(generateBuildCharacterFileSettings: _*) + .settings( + publishArtifact := false, + publish := {}, + publishLocal := {}, + commands ++= ScriptCommands.all + ) .aggregate(library, forkjoin, reflect, compiler, interactive, repl, replJline, replJlineEmbedded, scaladoc, scalap, actors, partestExtras, junit, libraryAll, scalaDist).settings( sources in Compile := Seq.empty, diff --git a/project/ScriptCommands.scala b/project/ScriptCommands.scala new file mode 100644 index 0000000000..537990d985 --- /dev/null +++ b/project/ScriptCommands.scala @@ -0,0 +1,19 @@ +import sbt._ +import Keys._ +import complete.DefaultParsers._ + +/** Custom commands for use by the Jenkins scripts. This keeps the surface area and call syntax small. */ +object ScriptCommands { + def all = Seq(setupPublishCore) + + /** Set up the environment for `validate/publish-core`. The argument is the Artifactory snapshot repository URL. */ + def setupPublishCore = Command.single("setupPublishCore") { case (state, url) => + Project.extract(state).append(Seq( + VersionUtil.baseVersionSuffix in Global := "SHA-SNAPSHOT", + // Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088): + publishTo in Global := Some("scala-pr" at url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis), + publishArtifact in (Compile, packageDoc) in ThisBuild := false, + scalacOptions in Compile in ThisBuild += "-optimise" + ), state) + } +} diff --git a/project/VersionUtil.scala b/project/VersionUtil.scala index 71de772b08..f237c35c68 100644 --- a/project/VersionUtil.scala +++ b/project/VersionUtil.scala @@ -5,21 +5,29 @@ import java.io.FileInputStream import scala.collection.JavaConverters._ object VersionUtil { + lazy val baseVersion = settingKey[String]("The base version number from which all others are derived") + lazy val baseVersionSuffix = settingKey[String]("Identifies the kind of version to build") lazy val copyrightString = settingKey[String]("Copyright string.") lazy val versionProperties = settingKey[Versions]("Version properties.") lazy val generateVersionPropertiesFile = taskKey[File]("Generating version properties file.") + lazy val generateBuildCharacterPropertiesFile = taskKey[File]("Generating buildcharacter.properties file.") - lazy val versionPropertiesSettings = Seq[Setting[_]]( - versionProperties := versionPropertiesImpl.value + lazy val globalVersionSettings = Seq[Setting[_]]( + // Set the version properties globally (they are the same for all projects) + versionProperties in Global := versionPropertiesImpl.value, + version in Global := versionProperties.value.mavenVersion ) lazy val generatePropertiesFileSettings = Seq[Setting[_]]( copyrightString := "Copyright 2002-2015, LAMP/EPFL", resourceGenerators in Compile += generateVersionPropertiesFile.map(file => Seq(file)).taskValue, - versionProperties := versionPropertiesImpl.value, generateVersionPropertiesFile := generateVersionPropertiesFileImpl.value ) + lazy val generateBuildCharacterFileSettings = Seq[Setting[_]]( + generateBuildCharacterPropertiesFile := generateBuildCharacterPropertiesFileImpl.value + ) + case class Versions(canonicalVersion: String, mavenVersion: String, osgiVersion: String, commitSha: String, commitDate: String, isRelease: Boolean) { val githubTree = if(isRelease) "v" + mavenVersion @@ -28,30 +36,36 @@ object VersionUtil { override def toString = s"Canonical: $canonicalVersion, Maven: $mavenVersion, OSGi: $osgiVersion, github: $githubTree" - def toProperties: Properties = { - val props = new Properties - props.put("version.number", canonicalVersion) - props.put("maven.version.number", mavenVersion) - props.put("osgi.version.number", osgiVersion) - props - } + def toMap: Map[String, String] = Map( + "version.number" -> canonicalVersion, + "maven.version.number" -> mavenVersion, + "osgi.version.number" -> osgiVersion + ) } - lazy val versionPropertiesImpl: Def.Initialize[Versions] = Def.setting { - /** Regexp that splits version number split into two parts: version and suffix. - * Examples of how the split is performed: - * - * "2.11.5": ("2.11.5", null) - * "2.11.5-acda7a": ("2.11.5", "-acda7a") - * "2.11.5-SNAPSHOT": ("2.11.5", "-SNAPSHOT") */ - val versionSplitted = """([\w+\.]+)(-[\w+\.]+)??""".r - - val versionSplitted(ver, suffixOrNull) = version.value - - val osgiSuffix = suffixOrNull match { - case null => "-VFINAL" - case "-SNAPSHOT" => "" - case suffixStr => suffixStr + /** Compute the canonical, Maven and OSGi version number from `baseVersion` and `baseVersionSuffix`. + * Examples of the generated versions: + * + * ("2.11.8", "SNAPSHOT" ) -> ("2.11.8-20151215-133023-7559aed3c5", "2.11.8-SNAPSHOT", "2.11.8.v20151215-133023-7559aed3c5") + * ("2.11.8", "SHA-SNAPSHOT") -> ("2.11.8-20151215-133023-7559aed3c5", "2.11.8-7559aed3c5-SNAPSHOT", "2.11.8.v20151215-133023-7559aed3c5") + * ("2.11.8", "" ) -> ("2.11.8", "2.11.8", "2.11.8.v20151215-133023-VFINAL-7559aed3c5") + * ("2.11.8", "M3" ) -> ("2.11.8-M3", "2.11.8-M3", "2.11.8.v20151215-133023-M3-7559aed3c5") + * ("2.11.8", "RC4" ) -> ("2.11.8-RC4", "2.11.8-RC4", "2.11.8.v20151215-133023-RC4-7559aed3c5") + * ("2.11.8-RC4", "SPLIT" ) -> ("2.11.8-RC4", "2.11.8-RC4", "2.11.8.v20151215-133023-RC4-7559aed3c5") + * + * A `baseVersionSuffix` of "SNAPSHOT" is the default, which is used for local snapshot builds. The PR validation + * job uses "SHA-SNAPSHOT". An empty suffix is used for releases. All other suffix values are treated as RC / + * milestone builds. The special suffix value "SPLIT" is used to split the real suffix off from `baseVersion` + * instead and then apply the usual logic. */ + private lazy val versionPropertiesImpl: Def.Initialize[Versions] = Def.setting { + + val (base, suffix) = { + val (b, s) = (baseVersion.value, baseVersionSuffix.value) + if(s == "SPLIT") { + val split = """([\w+\.]+)(-[\w+\.]+)??""".r + val split(b2, sOrNull) = b + (b2, Option(sOrNull).map(_.drop(1)).getOrElse("")) + } else (b, s) } def executeTool(tool: String) = { @@ -62,24 +76,31 @@ object VersionUtil { Process(cmd).lines.head } - val commitDate = executeTool("get-scala-commit-date") - val commitSha = executeTool("get-scala-commit-sha") + val date = executeTool("get-scala-commit-date") + val sha = executeTool("get-scala-commit-sha").substring(0, 7) // The script produces 10 digits at the moment - Versions( - canonicalVersion = s"$ver-$commitDate-$commitSha", - mavenVersion = s"${version.value}", - osgiVersion = s"$ver.v$commitDate$osgiSuffix-$commitSha", - commitSha = commitSha, - commitDate = commitDate, - isRelease = !osgiSuffix.isEmpty - ) + val (canonicalV, mavenV, osgiV, release) = suffix match { + case "SNAPSHOT" => (s"$base-$date-$sha", s"$base-SNAPSHOT", s"$base.v$date-$sha", false) + case "SHA-SNAPSHOT" => (s"$base-$date-$sha", s"$base-$sha-SNAPSHOT", s"$base.v$date-$sha", false) + case "" => (s"$base", s"$base", s"$base.v$date-VFINAL-$sha", true) + case suffix => (s"$base-$suffix", s"$base-$suffix", s"$base.v$date-$suffix-$sha", true) + } + + Versions(canonicalV, mavenV, osgiV, sha, date, release) } - lazy val generateVersionPropertiesFileImpl: Def.Initialize[Task[File]] = Def.task { - val props = versionProperties.value.toProperties - val propFile = (resourceManaged in Compile).value / s"${thisProject.value.id}.properties" - props.put("copyright.string", copyrightString.value) + private lazy val generateVersionPropertiesFileImpl: Def.Initialize[Task[File]] = Def.task { + writeProps(versionProperties.value.toMap + ("copyright.string" -> copyrightString.value), + (resourceManaged in Compile).value / s"${thisProject.value.id}.properties") + } + + private lazy val generateBuildCharacterPropertiesFileImpl: Def.Initialize[Task[File]] = Def.task { + writeProps(versionProperties.value.toMap, (baseDirectory in ThisBuild).value / "buildcharacter.properties") + } + private def writeProps(m: Map[String, String], propFile: File): File = { + val props = new Properties + m.foreach { case (k, v) => props.put(k, v) } // unfortunately, this will write properties in arbitrary order // this makes it harder to test for stability of generated artifacts // consider using https://github.com/etiennestuder/java-ordered-properties diff --git a/scripts/common b/scripts/common index b075469379..35199992bc 100644 --- a/scripts/common +++ b/scripts/common @@ -18,6 +18,9 @@ IVY_CACHE="$WORKSPACE/.ivy2" mkdir -p $IVY_CACHE rm -rf $IVY_CACHE/cache/org.scala-lang +SBT_CMD=${sbtCmd-sbt} +SBT_CMD="$SBT_CMD -sbt-version 0.13.9" + # temp dir where all 'non-build' operation are performed TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX) TMP_DIR="${TMP_ROOT_DIR}/tmp" diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap index 8d04e7fc79..cbb8e1026b 100755 --- a/scripts/jobs/integrate/bootstrap +++ b/scripts/jobs/integrate/bootstrap @@ -22,7 +22,7 @@ # - To prevent staging on sonatype (for testing), set publishToSonatype to anything but "yes" # - Note: After building a release, the jenkins job provides an updated versions.properties file as artifact. # Put this file in the Scala repo and create a pull request, and also update the file build.number. -# +# # - Otherwise, a nightly release is built: # - version number is read from the build.number file, extended with -$sha-nightly @@ -73,7 +73,7 @@ # Requirements -# - sbtCmd must point to sbt from sbt-extras +# - SBT_CMD must point to sbt from sbt-extras # - ~/.sonatype-curl, ~/.m2/settings.xml, ~/.credentials, ~/.credentials-sonatype, ~/.credentials-private-repo # as defined by https://github.com/scala/scala-jenkins-infra/tree/master/templates/default # - ~/.sbt/0.13/plugins/gpg.sbt with: @@ -89,10 +89,6 @@ publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"} publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"} publishLockerPrivateTask=${publishLockerPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built locker before) -sbtCmd=${sbtCmd-sbt} # TESTING (this is a marker for defaults to change when testing locally: should be sbtx on my mac) - -sbtCmd="$sbtCmd -sbt-version 0.13.8" - forceRebuild=${forceRebuild-no} antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak) @@ -180,8 +176,8 @@ function st_stagingRepoClose() { sbtArgs="-no-colors -ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13" sbtBuild() { - echo "### sbtBuild: "$sbtCmd $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" - $sbtCmd $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1 + echo "### sbtBuild: "$SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" + $SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1 } sbtResolve() { @@ -189,8 +185,8 @@ sbtResolve() { touch build.sbt # Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin. cross=${4-binary} - echo "### sbtResolve: $sbtCmd $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" - $sbtCmd $sbtArgs "${scalaVersionTasks[@]}" \ + echo "### sbtResolve: $SBT_CMD $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross" + $SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" \ "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \ 'show update' >> $baseDir/logs/resolution 2>&1 } @@ -257,7 +253,7 @@ buildContinuations() { else update scala scala-continuations $CONTINUATIONS_REF && gfxd - $sbtCmd $sbtArgs 'project plugin' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ + $SBT_CMD $sbtArgs 'project plugin' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ 'set version := "'$CONTINUATIONS_VER'"' $clean "compile:package" test "${buildTasks[@]}" # https://github.com/scala/scala-continuations/pull/4 CONT_PLUG_BUILT="yes" fi @@ -266,7 +262,7 @@ buildContinuations() { then echo "Found scala-continuations-library $CONTINUATIONS_VER; not building." else update scala scala-continuations $CONTINUATIONS_REF && gfxd - $sbtCmd $sbtArgs 'project library' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ + $SBT_CMD $sbtArgs 'project library' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \ 'set version := "'$CONTINUATIONS_VER'"' $clean test "${buildTasks[@]}" CONT_LIB_BUILT="yes" fi diff --git a/scripts/jobs/validate/publish-core b/scripts/jobs/validate/publish-core index 9dff5a34b0..bb0056722d 100755 --- a/scripts/jobs/validate/publish-core +++ b/scripts/jobs/validate/publish-core @@ -15,17 +15,8 @@ case $prDryRun in mkdir -p build/pack ; mkdir -p dists/maven/latest ;; *) - sha=$(git rev-parse HEAD) # TODO: warn if $repo_ref != $sha (we shouldn't do PR validation using symbolic gitrefs) - echo "sha/repo_ref == $sha/$repo_ref ?" - - parseScalaProperties build.number - - ./pull-binary-libs.sh - # "noyoudont" is there juuuust in case - antDeployArgs="-Dmaven.version.suffix=\"-${sha:0:7}-SNAPSHOT\" -Dremote.snapshot.repository=$prRepoUrl -Drepository.credentials.id=pr-scala -Dremote.release.repository=noyoudont" - echo ">>> Getting Scala version number." - ant -q $antDeployArgs init + $SBT_CMD "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile parseScalaProperties buildcharacter.properties # produce maven_version_number echo ">>> Checking availability of Scala ${maven_version_number} in $prRepoUrl." @@ -36,7 +27,7 @@ case $prDryRun in if $libraryAvailable && $reflectAvailable && $compilerAvailable; then echo "Scala core already built!" else - ant $antDeployArgs $antBuildArgs publish-opt-nodocs + $SBT_CMD "setupPublishCore $prRepoUrl" $antBuildArgs publish fi mv buildcharacter.properties jenkins.properties # parsed by the jenkins job |