diff options
author | Yanbo Liang <ybliang8@gmail.com> | 2016-12-08 06:19:38 -0800 |
---|---|---|
committer | Yanbo Liang <ybliang8@gmail.com> | 2016-12-08 06:19:38 -0800 |
commit | 9bf8f3cd4f62f921c32fb50b8abf49576a80874f (patch) | |
tree | 37a11c334001a981d5a87c5d0fefec67d2ef4889 /examples/src | |
parent | b47b892e4579b7b06b4b2837ee4b614e517789f9 (diff) | |
download | spark-9bf8f3cd4f62f921c32fb50b8abf49576a80874f.tar.gz spark-9bf8f3cd4f62f921c32fb50b8abf49576a80874f.tar.bz2 spark-9bf8f3cd4f62f921c32fb50b8abf49576a80874f.zip |
[SPARK-18325][SPARKR][ML] SparkR ML wrappers example code and user guide
## What changes were proposed in this pull request?
* Add all R examples for ML wrappers which were added during 2.1 release cycle.
* Split the whole ```ml.R``` example file into individual example for each algorithm, which will be convenient for users to rerun them.
* Add corresponding examples to ML user guide.
* Update ML section of SparkR user guide.
Note: MLlib Scala/Java/Python examples will be consistent, however, SparkR examples may different from them, since R users may use the algorithms in a different way, for example, using R ```formula``` to specify ```featuresCol``` and ```labelCol```.
## How was this patch tested?
Run all examples manually.
Author: Yanbo Liang <ybliang8@gmail.com>
Closes #16148 from yanboliang/spark-18325.
Diffstat (limited to 'examples/src')
-rw-r--r-- | examples/src/main/r/ml.R | 148 | ||||
-rw-r--r-- | examples/src/main/r/ml/als.R | 45 | ||||
-rw-r--r-- | examples/src/main/r/ml/gaussianMixture.R | 42 | ||||
-rw-r--r-- | examples/src/main/r/ml/gbt.R | 63 | ||||
-rw-r--r-- | examples/src/main/r/ml/glm.R | 57 | ||||
-rw-r--r-- | examples/src/main/r/ml/isoreg.R | 42 | ||||
-rw-r--r-- | examples/src/main/r/ml/kmeans.R | 44 | ||||
-rw-r--r-- | examples/src/main/r/ml/kstest.R | 39 | ||||
-rw-r--r-- | examples/src/main/r/ml/lda.R | 46 | ||||
-rw-r--r-- | examples/src/main/r/ml/logit.R | 63 | ||||
-rw-r--r-- | examples/src/main/r/ml/ml.R | 65 | ||||
-rw-r--r-- | examples/src/main/r/ml/mlp.R | 48 | ||||
-rw-r--r-- | examples/src/main/r/ml/naiveBayes.R | 41 | ||||
-rw-r--r-- | examples/src/main/r/ml/randomForest.R | 63 | ||||
-rw-r--r-- | examples/src/main/r/ml/survreg.R | 43 |
15 files changed, 701 insertions, 148 deletions
diff --git a/examples/src/main/r/ml.R b/examples/src/main/r/ml.R deleted file mode 100644 index a8a1274ac9..0000000000 --- a/examples/src/main/r/ml.R +++ /dev/null @@ -1,148 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# To run this example use -# ./bin/spark-submit examples/src/main/r/ml.R - -# Load SparkR library into your R session -library(SparkR) - -# Initialize SparkSession -sparkR.session(appName = "SparkR-ML-example") - -############################ spark.glm and glm ############################################## -# $example on:glm$ -irisDF <- suppressWarnings(createDataFrame(iris)) -# Fit a generalized linear model of family "gaussian" with spark.glm -gaussianDF <- irisDF -gaussianTestDF <- irisDF -gaussianGLM <- spark.glm(gaussianDF, Sepal_Length ~ Sepal_Width + Species, family = "gaussian") - -# Model summary -summary(gaussianGLM) - -# Prediction -gaussianPredictions <- predict(gaussianGLM, gaussianTestDF) -showDF(gaussianPredictions) - -# Fit a generalized linear model with glm (R-compliant) -gaussianGLM2 <- glm(Sepal_Length ~ Sepal_Width + Species, gaussianDF, family = "gaussian") -summary(gaussianGLM2) - -# Fit a generalized linear model of family "binomial" with spark.glm -binomialDF <- filter(irisDF, irisDF$Species != "setosa") -binomialTestDF <- binomialDF -binomialGLM <- spark.glm(binomialDF, Species ~ Sepal_Length + Sepal_Width, family = "binomial") - -# Model summary -summary(binomialGLM) - -# Prediction -binomialPredictions <- predict(binomialGLM, binomialTestDF) -showDF(binomialPredictions) -# $example off:glm$ -############################ spark.survreg ############################################## -# $example on:survreg$ -# Use the ovarian dataset available in R survival package -library(survival) - -# Fit an accelerated failure time (AFT) survival regression model with spark.survreg -ovarianDF <- suppressWarnings(createDataFrame(ovarian)) -aftDF <- ovarianDF -aftTestDF <- ovarianDF -aftModel <- spark.survreg(aftDF, Surv(futime, fustat) ~ ecog_ps + rx) - -# Model summary -summary(aftModel) - -# Prediction -aftPredictions <- predict(aftModel, aftTestDF) -showDF(aftPredictions) -# $example off:survreg$ -############################ spark.naiveBayes ############################################## -# $example on:naiveBayes$ -# Fit a Bernoulli naive Bayes model with spark.naiveBayes -titanic <- as.data.frame(Titanic) -titanicDF <- createDataFrame(titanic[titanic$Freq > 0, -5]) -nbDF <- titanicDF -nbTestDF <- titanicDF -nbModel <- spark.naiveBayes(nbDF, Survived ~ Class + Sex + Age) - -# Model summary -summary(nbModel) - -# Prediction -nbPredictions <- predict(nbModel, nbTestDF) -showDF(nbPredictions) -# $example off:naiveBayes$ -############################ spark.kmeans ############################################## -# $example on:kmeans$ -# Fit a k-means model with spark.kmeans -irisDF <- suppressWarnings(createDataFrame(iris)) -kmeansDF <- irisDF -kmeansTestDF <- irisDF -kmeansModel <- spark.kmeans(kmeansDF, ~ Sepal_Length + Sepal_Width + Petal_Length + Petal_Width, - k = 3) - -# Model summary -summary(kmeansModel) - -# Get fitted result from the k-means model -showDF(fitted(kmeansModel)) - -# Prediction -kmeansPredictions <- predict(kmeansModel, kmeansTestDF) -showDF(kmeansPredictions) -# $example off:kmeans$ -############################ model read/write ############################################## -# $example on:read_write$ -irisDF <- suppressWarnings(createDataFrame(iris)) -# Fit a generalized linear model of family "gaussian" with spark.glm -gaussianDF <- irisDF -gaussianTestDF <- irisDF -gaussianGLM <- spark.glm(gaussianDF, Sepal_Length ~ Sepal_Width + Species, family = "gaussian") - -# Save and then load a fitted MLlib model -modelPath <- tempfile(pattern = "ml", fileext = ".tmp") -write.ml(gaussianGLM, modelPath) -gaussianGLM2 <- read.ml(modelPath) - -# Check model summary -summary(gaussianGLM2) - -# Check model prediction -gaussianPredictions <- predict(gaussianGLM2, gaussianTestDF) -showDF(gaussianPredictions) - -unlink(modelPath) -# $example off:read_write$ -############################ fit models with spark.lapply ##################################### - -# Perform distributed training of multiple models with spark.lapply -families <- c("gaussian", "poisson") -train <- function(family) { - model <- glm(Sepal.Length ~ Sepal.Width + Species, iris, family = family) - summary(model) -} -model.summaries <- spark.lapply(families, train) - -# Print the summary of each model -print(model.summaries) - - -# Stop the SparkSession now -sparkR.session.stop() diff --git a/examples/src/main/r/ml/als.R b/examples/src/main/r/ml/als.R new file mode 100644 index 0000000000..383bbba190 --- /dev/null +++ b/examples/src/main/r/ml/als.R @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/als.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-als-example") + +# $example on$ +# Load training data +data <- list(list(0, 0, 4.0), list(0, 1, 2.0), list(1, 1, 3.0), + list(1, 2, 4.0), list(2, 1, 1.0), list(2, 2, 5.0)) +df <- createDataFrame(data, c("userId", "movieId", "rating")) +training <- df +test <- df + +# Fit a recommendation model using ALS with spark.als +model <- spark.als(training, maxIter = 5, regParam = 0.01, userCol = "userId", + itemCol = "movieId", ratingCol = "rating") + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off$ diff --git a/examples/src/main/r/ml/gaussianMixture.R b/examples/src/main/r/ml/gaussianMixture.R new file mode 100644 index 0000000000..54b69acc83 --- /dev/null +++ b/examples/src/main/r/ml/gaussianMixture.R @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/gaussianMixture.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-gaussianMixture-example") + +# $example on$ +# Load training data +df <- read.df("data/mllib/sample_kmeans_data.txt", source = "libsvm") +training <- df +test <- df + +# Fit a gaussian mixture clustering model with spark.gaussianMixture +model <- spark.gaussianMixture(training, ~ features, k = 2) + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off$ diff --git a/examples/src/main/r/ml/gbt.R b/examples/src/main/r/ml/gbt.R new file mode 100644 index 0000000000..be16c2aa66 --- /dev/null +++ b/examples/src/main/r/ml/gbt.R @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/gbt.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-gbt-example") + +# GBT classification model + +# $example on:classification$ +# Load training data +df <- read.df("data/mllib/sample_libsvm_data.txt", source = "libsvm") +training <- df +test <- df + +# Fit a GBT classification model with spark.gbt +model <- spark.gbt(training, label ~ features, "classification", maxIter = 10) + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off:classification$ + +# GBT regression model + +# $example on:regression$ +# Load training data +df <- read.df("data/mllib/sample_linear_regression_data.txt", source = "libsvm") +training <- df +test <- df + +# Fit a GBT regression model with spark.gbt +model <- spark.gbt(training, label ~ features, "regression", maxIter = 10) + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off:regression$ diff --git a/examples/src/main/r/ml/glm.R b/examples/src/main/r/ml/glm.R new file mode 100644 index 0000000000..599071790a --- /dev/null +++ b/examples/src/main/r/ml/glm.R @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/glm.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-glm-example") + +# $example on$ +irisDF <- suppressWarnings(createDataFrame(iris)) +# Fit a generalized linear model of family "gaussian" with spark.glm +gaussianDF <- irisDF +gaussianTestDF <- irisDF +gaussianGLM <- spark.glm(gaussianDF, Sepal_Length ~ Sepal_Width + Species, family = "gaussian") + +# Model summary +summary(gaussianGLM) + +# Prediction +gaussianPredictions <- predict(gaussianGLM, gaussianTestDF) +showDF(gaussianPredictions) + +# Fit a generalized linear model with glm (R-compliant) +gaussianGLM2 <- glm(Sepal_Length ~ Sepal_Width + Species, gaussianDF, family = "gaussian") +summary(gaussianGLM2) + +# Fit a generalized linear model of family "binomial" with spark.glm +# Note: Filter out "setosa" from label column (two labels left) to match "binomial" family. +binomialDF <- filter(irisDF, irisDF$Species != "setosa") +binomialTestDF <- binomialDF +binomialGLM <- spark.glm(binomialDF, Species ~ Sepal_Length + Sepal_Width, family = "binomial") + +# Model summary +summary(binomialGLM) + +# Prediction +binomialPredictions <- predict(binomialGLM, binomialTestDF) +showDF(binomialPredictions) +# $example off$ diff --git a/examples/src/main/r/ml/isoreg.R b/examples/src/main/r/ml/isoreg.R new file mode 100644 index 0000000000..75dce97ed9 --- /dev/null +++ b/examples/src/main/r/ml/isoreg.R @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/isoreg.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-isoreg-example") + +# $example on$ +# Load training data +df <- read.df("data/mllib/sample_isotonic_regression_libsvm_data.txt", source = "libsvm") +training <- df +test <- df + +# Fit an isotonic regression model with spark.isoreg +model <- spark.isoreg(training, label ~ features, isotonic = FALSE) + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off$ diff --git a/examples/src/main/r/ml/kmeans.R b/examples/src/main/r/ml/kmeans.R new file mode 100644 index 0000000000..043b21b038 --- /dev/null +++ b/examples/src/main/r/ml/kmeans.R @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/kmeans.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-kmeans-example") + +# $example on$ +# Fit a k-means model with spark.kmeans +irisDF <- suppressWarnings(createDataFrame(iris)) +kmeansDF <- irisDF +kmeansTestDF <- irisDF +kmeansModel <- spark.kmeans(kmeansDF, ~ Sepal_Length + Sepal_Width + Petal_Length + Petal_Width, + k = 3) + +# Model summary +summary(kmeansModel) + +# Get fitted result from the k-means model +showDF(fitted(kmeansModel)) + +# Prediction +kmeansPredictions <- predict(kmeansModel, kmeansTestDF) +showDF(kmeansPredictions) +# $example off$ diff --git a/examples/src/main/r/ml/kstest.R b/examples/src/main/r/ml/kstest.R new file mode 100644 index 0000000000..12625f7d3e --- /dev/null +++ b/examples/src/main/r/ml/kstest.R @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/kstest.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-kstest-example") + +# $example on$ +# Load training data +data <- data.frame(test = c(0.1, 0.15, 0.2, 0.3, 0.25, -1, -0.5)) +df <- createDataFrame(data) +training <- df +test <- df + +# Conduct the two-sided Kolmogorov-Smirnov (KS) test with spark.kstest +model <- spark.kstest(df, "test", "norm") + +# Model summary +summary(model) +# $example off$ diff --git a/examples/src/main/r/ml/lda.R b/examples/src/main/r/ml/lda.R new file mode 100644 index 0000000000..7b187d155a --- /dev/null +++ b/examples/src/main/r/ml/lda.R @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/lda.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-lda-example") + +# $example on$ +# Load training data +df <- read.df("data/mllib/sample_lda_libsvm_data.txt", source = "libsvm") +training <- df +test <- df + +# Fit a latent dirichlet allocation model with spark.lda +model <- spark.lda(training, k = 10, maxIter = 10) + +# Model summary +summary(model) + +# Posterior probabilities +posterior <- spark.posterior(model, test) +showDF(posterior) + +# The log perplexity of the LDA model +logPerplexity <- spark.perplexity(model, test) +print(paste0("The upper bound bound on perplexity: ", logPerplexity)) +# $example off$ diff --git a/examples/src/main/r/ml/logit.R b/examples/src/main/r/ml/logit.R new file mode 100644 index 0000000000..a2ac882ed0 --- /dev/null +++ b/examples/src/main/r/ml/logit.R @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/logit.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-logit-example") + +# Binomial logistic regression + +# $example on:binomial$ +# Load training data +df <- read.df("data/mllib/sample_libsvm_data.txt", source = "libsvm") +training <- df +test <- df + +# Fit an binomial logistic regression model with spark.logit +model <- spark.logit(training, label ~ features, maxIter = 10, regParam = 0.3, elasticNetParam = 0.8) + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off:binomial$ + +# Multinomial logistic regression + +# $example on:multinomial$ +# Load training data +df <- read.df("data/mllib/sample_multiclass_classification_data.txt", source = "libsvm") +training <- df +test <- df + +# Fit a multinomial logistic regression model with spark.logit +model <- spark.logit(training, label ~ features, maxIter = 10, regParam = 0.3, elasticNetParam = 0.8) + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off:multinomial$ diff --git a/examples/src/main/r/ml/ml.R b/examples/src/main/r/ml/ml.R new file mode 100644 index 0000000000..d601590c22 --- /dev/null +++ b/examples/src/main/r/ml/ml.R @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/ml.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-example") + +############################ model read/write ############################################## +# $example on:read_write$ +irisDF <- suppressWarnings(createDataFrame(iris)) +# Fit a generalized linear model of family "gaussian" with spark.glm +gaussianDF <- irisDF +gaussianTestDF <- irisDF +gaussianGLM <- spark.glm(gaussianDF, Sepal_Length ~ Sepal_Width + Species, family = "gaussian") + +# Save and then load a fitted MLlib model +modelPath <- tempfile(pattern = "ml", fileext = ".tmp") +write.ml(gaussianGLM, modelPath) +gaussianGLM2 <- read.ml(modelPath) + +# Check model summary +summary(gaussianGLM2) + +# Check model prediction +gaussianPredictions <- predict(gaussianGLM2, gaussianTestDF) +showDF(gaussianPredictions) + +unlink(modelPath) +# $example off:read_write$ + +############################ fit models with spark.lapply ##################################### +# Perform distributed training of multiple models with spark.lapply +costs <- exp(seq(from = log(1), to = log(1000), length.out = 5)) +train <- function(cost) { + stopifnot(requireNamespace("e1071", quietly = TRUE)) + model <- e1071::svm(Species ~ ., data = iris, cost = cost) + summary(model) +} + +model.summaries <- spark.lapply(costs, train) + +# Print the summary of each model +print(model.summaries) + +# Stop the SparkSession now +sparkR.session.stop() diff --git a/examples/src/main/r/ml/mlp.R b/examples/src/main/r/ml/mlp.R new file mode 100644 index 0000000000..d28fc069bd --- /dev/null +++ b/examples/src/main/r/ml/mlp.R @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/mlp.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-mlp-example") + +# $example on$ +# Load training data +df <- read.df("data/mllib/sample_multiclass_classification_data.txt", source = "libsvm") +training <- df +test <- df + +# specify layers for the neural network: +# input layer of size 4 (features), two intermediate of size 5 and 4 +# and output of size 3 (classes) +layers = c(4, 5, 4, 3) + +# Fit a multi-layer perceptron neural network model with spark.mlp +model <- spark.mlp(training, label ~ features, maxIter = 100, + layers = layers, blockSize = 128, seed = 1234) + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off$ diff --git a/examples/src/main/r/ml/naiveBayes.R b/examples/src/main/r/ml/naiveBayes.R new file mode 100644 index 0000000000..9c416599b4 --- /dev/null +++ b/examples/src/main/r/ml/naiveBayes.R @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/naiveBayes.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-naiveBayes-example") + +# $example on$ +# Fit a Bernoulli naive Bayes model with spark.naiveBayes +titanic <- as.data.frame(Titanic) +titanicDF <- createDataFrame(titanic[titanic$Freq > 0, -5]) +nbDF <- titanicDF +nbTestDF <- titanicDF +nbModel <- spark.naiveBayes(nbDF, Survived ~ Class + Sex + Age) + +# Model summary +summary(nbModel) + +# Prediction +nbPredictions <- predict(nbModel, nbTestDF) +showDF(nbPredictions) +# $example off$ diff --git a/examples/src/main/r/ml/randomForest.R b/examples/src/main/r/ml/randomForest.R new file mode 100644 index 0000000000..d1b96b62a0 --- /dev/null +++ b/examples/src/main/r/ml/randomForest.R @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/randomForest.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-randomForest-example") + +# Random forest classification model + +# $example on:classification$ +# Load training data +df <- read.df("data/mllib/sample_libsvm_data.txt", source = "libsvm") +training <- df +test <- df + +# Fit a random forest classification model with spark.randomForest +model <- spark.randomForest(training, label ~ features, "classification", numTrees = 10) + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off:classification$ + +# Random forest regression model + +# $example on:regression$ +# Load training data +df <- read.df("data/mllib/sample_linear_regression_data.txt", source = "libsvm") +training <- df +test <- df + +# Fit a random forest regression model with spark.randomForest +model <- spark.randomForest(training, label ~ features, "regression", numTrees = 10) + +# Model summary +summary(model) + +# Prediction +predictions <- predict(model, test) +showDF(predictions) +# $example off:regression$ diff --git a/examples/src/main/r/ml/survreg.R b/examples/src/main/r/ml/survreg.R new file mode 100644 index 0000000000..f728b8b5d8 --- /dev/null +++ b/examples/src/main/r/ml/survreg.R @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To run this example use +# ./bin/spark-submit examples/src/main/r/ml/survreg.R + +# Load SparkR library into your R session +library(SparkR) + +# Initialize SparkSession +sparkR.session(appName = "SparkR-ML-survreg-example") + +# $example on$ +# Use the ovarian dataset available in R survival package +library(survival) + +# Fit an accelerated failure time (AFT) survival regression model with spark.survreg +ovarianDF <- suppressWarnings(createDataFrame(ovarian)) +aftDF <- ovarianDF +aftTestDF <- ovarianDF +aftModel <- spark.survreg(aftDF, Surv(futime, fustat) ~ ecog_ps + rx) + +# Model summary +summary(aftModel) + +# Prediction +aftPredictions <- predict(aftModel, aftTestDF) +showDF(aftPredictions) +# $example off$ |