diff options
author | hyukjinkwon <gurwls223@gmail.com> | 2016-09-08 08:26:59 -0700 |
---|---|---|
committer | Shivaram Venkataraman <shivaram@cs.berkeley.edu> | 2016-09-08 08:26:59 -0700 |
commit | 78d5d4dd5ce5a537ed04cd1bf242c9e9ea2c391a (patch) | |
tree | 0713f74a384a96cbb42a74f74ce1c1e05b2338af /dev/appveyor-install-dependencies.ps1 | |
parent | f0d21b7f90cdcce353ab6fc279b9cc376e46e536 (diff) | |
download | spark-78d5d4dd5ce5a537ed04cd1bf242c9e9ea2c391a.tar.gz spark-78d5d4dd5ce5a537ed04cd1bf242c9e9ea2c391a.tar.bz2 spark-78d5d4dd5ce5a537ed04cd1bf242c9e9ea2c391a.zip |
[SPARK-17200][PROJECT INFRA][BUILD][SPARKR] Automate building and testing on Windows (currently SparkR only)
## What changes were proposed in this pull request?
This PR adds the build automation on Windows with [AppVeyor](https://www.appveyor.com/) CI tool.
Currently, this only runs the tests for SparkR as we have been having some issues with testing Windows-specific PRs (e.g. https://github.com/apache/spark/pull/14743 and https://github.com/apache/spark/pull/13165) and hard time to verify this.
One concern is, this build is dependent on [steveloughran/winutils](https://github.com/steveloughran/winutils) for pre-built Hadoop bin package (who is a Hadoop PMC member).
## How was this patch tested?
Manually, https://ci.appveyor.com/project/HyukjinKwon/spark/build/88-SPARK-17200-build-profile
This takes roughly 40 mins.
Some tests are already being failed and this was found in https://github.com/apache/spark/pull/14743#issuecomment-241405287.
Author: hyukjinkwon <gurwls223@gmail.com>
Closes #14859 from HyukjinKwon/SPARK-17200-build.
Diffstat (limited to 'dev/appveyor-install-dependencies.ps1')
-rw-r--r-- | dev/appveyor-install-dependencies.ps1 | 126 |
1 files changed, 126 insertions, 0 deletions
diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 new file mode 100644 index 0000000000..087b8666cc --- /dev/null +++ b/dev/appveyor-install-dependencies.ps1 @@ -0,0 +1,126 @@ +<# +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +#> + +$CRAN = "https://cloud.r-project.org" + +Function InstallR { + if ( -not(Test-Path Env:\R_ARCH) ) { + $arch = "i386" + } + Else { + $arch = $env:R_ARCH + } + + $urlPath = "" + $latestVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + If ($rVer -ne $latestVer) { + $urlPath = ("old/" + $rVer + "/") + } + + $rurl = $CRAN + "/bin/windows/base/" + $urlPath + "R-" + $rVer + "-win.exe" + + # Downloading R + Start-FileDownload $rurl "R-win.exe" + + # Running R installer + Start-Process -FilePath .\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait + + $RDrive = "C:" + echo "R is now available on drive $RDrive" + + $env:PATH = $RDrive + '\R\bin\' + $arch + ';' + 'C:\MinGW\msys\1.0\bin;' + $env:PATH + + # Testing R installation + Rscript -e "sessionInfo()" +} + +Function InstallRtools { + $rtoolsver = $rToolsVer.Split('.')[0..1] -Join '' + $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" + + # Downloading Rtools + Start-FileDownload $rtoolsurl "Rtools-current.exe" + + # Running Rtools installer + Start-Process -FilePath .\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait + + $RtoolsDrive = "C:" + echo "Rtools is now available on drive $RtoolsDrive" + + if ( -not(Test-Path Env:\GCC_PATH) ) { + $gccPath = "gcc-4.6.3" + } + Else { + $gccPath = $env:GCC_PATH + } + $env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gccPath + '\bin;' + $env:PATH + $env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/' +} + +# create tools directory outside of Spark directory +$up = (Get-Item -Path ".." -Verbose).FullName +$tools = "$up\tools" +if (!(Test-Path $tools)) { + New-Item -ItemType Directory -Force -Path $tools | Out-Null +} + +# ========================== Maven +Push-Location $tools + +$mavenVer = "3.3.9" +Start-FileDownload "https://archive.apache.org/dist/maven/maven-3/$mavenVer/binaries/apache-maven-$mavenVer-bin.zip" "maven.zip" + +# extract +Invoke-Expression "7z.exe x maven.zip" + +# add maven to environment variables +$env:Path += ";$tools\apache-maven-$mavenVer\bin" +$env:M2_HOME = "$tools\apache-maven-$mavenVer" +$env:MAVEN_OPTS = "-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m" + +Pop-Location + +# ========================== Hadoop bin package +$hadoopVer = "2.6.0" +$hadoopPath = "$tools\hadoop" +if (!(Test-Path $hadoopPath)) { + New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null +} +Push-Location $hadoopPath + +Start-FileDownload "https://github.com/steveloughran/winutils/archive/master.zip" "winutils-master.zip" + +# extract +Invoke-Expression "7z.exe x winutils-master.zip" + +# add hadoop bin to environment variables +$env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer" + +Pop-Location + +# ========================== R +$rVer = "3.3.1" +$rToolsVer = "3.4.0" + +InstallR +InstallRtools + +$env:R_LIBS_USER = 'c:\RLibrary' +if ( -not(Test-Path $env:R_LIBS_USER) ) { + mkdir $env:R_LIBS_USER +} + |