aboutsummaryrefslogtreecommitdiff
path: root/dev/appveyor-install-dependencies.ps1
diff options
context:
space:
mode:
authorhyukjinkwon <gurwls223@gmail.com>2016-09-08 08:26:59 -0700
committerShivaram Venkataraman <shivaram@cs.berkeley.edu>2016-09-08 08:26:59 -0700
commit78d5d4dd5ce5a537ed04cd1bf242c9e9ea2c391a (patch)
tree0713f74a384a96cbb42a74f74ce1c1e05b2338af /dev/appveyor-install-dependencies.ps1
parentf0d21b7f90cdcce353ab6fc279b9cc376e46e536 (diff)
downloadspark-78d5d4dd5ce5a537ed04cd1bf242c9e9ea2c391a.tar.gz
spark-78d5d4dd5ce5a537ed04cd1bf242c9e9ea2c391a.tar.bz2
spark-78d5d4dd5ce5a537ed04cd1bf242c9e9ea2c391a.zip
[SPARK-17200][PROJECT INFRA][BUILD][SPARKR] Automate building and testing on Windows (currently SparkR only)
## What changes were proposed in this pull request? This PR adds the build automation on Windows with [AppVeyor](https://www.appveyor.com/) CI tool. Currently, this only runs the tests for SparkR as we have been having some issues with testing Windows-specific PRs (e.g. https://github.com/apache/spark/pull/14743 and https://github.com/apache/spark/pull/13165) and hard time to verify this. One concern is, this build is dependent on [steveloughran/winutils](https://github.com/steveloughran/winutils) for pre-built Hadoop bin package (who is a Hadoop PMC member). ## How was this patch tested? Manually, https://ci.appveyor.com/project/HyukjinKwon/spark/build/88-SPARK-17200-build-profile This takes roughly 40 mins. Some tests are already being failed and this was found in https://github.com/apache/spark/pull/14743#issuecomment-241405287. Author: hyukjinkwon <gurwls223@gmail.com> Closes #14859 from HyukjinKwon/SPARK-17200-build.
Diffstat (limited to 'dev/appveyor-install-dependencies.ps1')
-rw-r--r--dev/appveyor-install-dependencies.ps1126
1 files changed, 126 insertions, 0 deletions
diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1
new file mode 100644
index 0000000000..087b8666cc
--- /dev/null
+++ b/dev/appveyor-install-dependencies.ps1
@@ -0,0 +1,126 @@
+<#
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+#>
+
+$CRAN = "https://cloud.r-project.org"
+
+Function InstallR {
+ if ( -not(Test-Path Env:\R_ARCH) ) {
+ $arch = "i386"
+ }
+ Else {
+ $arch = $env:R_ARCH
+ }
+
+ $urlPath = ""
+ $latestVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version
+ If ($rVer -ne $latestVer) {
+ $urlPath = ("old/" + $rVer + "/")
+ }
+
+ $rurl = $CRAN + "/bin/windows/base/" + $urlPath + "R-" + $rVer + "-win.exe"
+
+ # Downloading R
+ Start-FileDownload $rurl "R-win.exe"
+
+ # Running R installer
+ Start-Process -FilePath .\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
+
+ $RDrive = "C:"
+ echo "R is now available on drive $RDrive"
+
+ $env:PATH = $RDrive + '\R\bin\' + $arch + ';' + 'C:\MinGW\msys\1.0\bin;' + $env:PATH
+
+ # Testing R installation
+ Rscript -e "sessionInfo()"
+}
+
+Function InstallRtools {
+ $rtoolsver = $rToolsVer.Split('.')[0..1] -Join ''
+ $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe"
+
+ # Downloading Rtools
+ Start-FileDownload $rtoolsurl "Rtools-current.exe"
+
+ # Running Rtools installer
+ Start-Process -FilePath .\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait
+
+ $RtoolsDrive = "C:"
+ echo "Rtools is now available on drive $RtoolsDrive"
+
+ if ( -not(Test-Path Env:\GCC_PATH) ) {
+ $gccPath = "gcc-4.6.3"
+ }
+ Else {
+ $gccPath = $env:GCC_PATH
+ }
+ $env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gccPath + '\bin;' + $env:PATH
+ $env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/'
+}
+
+# create tools directory outside of Spark directory
+$up = (Get-Item -Path ".." -Verbose).FullName
+$tools = "$up\tools"
+if (!(Test-Path $tools)) {
+ New-Item -ItemType Directory -Force -Path $tools | Out-Null
+}
+
+# ========================== Maven
+Push-Location $tools
+
+$mavenVer = "3.3.9"
+Start-FileDownload "https://archive.apache.org/dist/maven/maven-3/$mavenVer/binaries/apache-maven-$mavenVer-bin.zip" "maven.zip"
+
+# extract
+Invoke-Expression "7z.exe x maven.zip"
+
+# add maven to environment variables
+$env:Path += ";$tools\apache-maven-$mavenVer\bin"
+$env:M2_HOME = "$tools\apache-maven-$mavenVer"
+$env:MAVEN_OPTS = "-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"
+
+Pop-Location
+
+# ========================== Hadoop bin package
+$hadoopVer = "2.6.0"
+$hadoopPath = "$tools\hadoop"
+if (!(Test-Path $hadoopPath)) {
+ New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null
+}
+Push-Location $hadoopPath
+
+Start-FileDownload "https://github.com/steveloughran/winutils/archive/master.zip" "winutils-master.zip"
+
+# extract
+Invoke-Expression "7z.exe x winutils-master.zip"
+
+# add hadoop bin to environment variables
+$env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer"
+
+Pop-Location
+
+# ========================== R
+$rVer = "3.3.1"
+$rToolsVer = "3.4.0"
+
+InstallR
+InstallRtools
+
+$env:R_LIBS_USER = 'c:\RLibrary'
+if ( -not(Test-Path $env:R_LIBS_USER) ) {
+ mkdir $env:R_LIBS_USER
+}
+