From 7e8279fde176b08687adf2b410693b35cfbd4b46 Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Wed, 27 Jul 2016 15:37:38 +0200 Subject: [SPARK-15254][DOC] Improve ML pipeline Cross Validation Scaladoc & PyDoc ## What changes were proposed in this pull request? Updated ML pipeline Cross Validation Scaladoc & PyDoc. ## How was this patch tested? Documentation update (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) Author: krishnakalyan3 Closes #13894 from krishnakalyan3/kfold-cv. --- python/pyspark/ml/tuning.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'python') diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py index 298314d46c..7f967e5463 100644 --- a/python/pyspark/ml/tuning.py +++ b/python/pyspark/ml/tuning.py @@ -143,7 +143,13 @@ class ValidatorParams(HasSeed): class CrossValidator(Estimator, ValidatorParams): """ - K-fold cross validation. + + K-fold cross validation performs model selection by splitting the dataset into a set of + non-overlapping randomly partitioned folds which are used as separate training and test datasets + e.g., with k=3 folds, K-fold cross validation will generate 3 (training, test) dataset pairs, + each of which uses 2/3 of the data for training and 1/3 for testing. Each fold is used as the + test set exactly once. + >>> from pyspark.ml.classification import LogisticRegression >>> from pyspark.ml.evaluation import BinaryClassificationEvaluator @@ -260,7 +266,10 @@ class CrossValidator(Estimator, ValidatorParams): class CrossValidatorModel(Model, ValidatorParams): """ - Model from k-fold cross validation. + + CrossValidatorModel contains the model with the highest average cross-validation + metric across folds and uses this model to transform input data. CrossValidatorModel + also tracks the metrics for each param map evaluated. .. versionadded:: 1.4.0 """ -- cgit v1.2.3