diff options
author | Zheng RuiFeng <ruifengz@foxmail.com> | 2016-05-11 09:53:36 +0200 |
---|---|---|
committer | Nick Pentreath <nickp@za.ibm.com> | 2016-05-11 09:53:36 +0200 |
commit | ad1a8466e9c10fbe8b455dba17b16973f92ebc15 (patch) | |
tree | 7da9c4df44d1774c2834b9d11cbc4f55aa3c8309 /examples/src/main/python | |
parent | 875ef764280428acd095aec1834fee0ddad08611 (diff) | |
download | spark-ad1a8466e9c10fbe8b455dba17b16973f92ebc15.tar.gz spark-ad1a8466e9c10fbe8b455dba17b16973f92ebc15.tar.bz2 spark-ad1a8466e9c10fbe8b455dba17b16973f92ebc15.zip |
[SPARK-15141][EXAMPLE][DOC] Update OneVsRest Examples
## What changes were proposed in this pull request?
1, Add python example for OneVsRest
2, remove args-parsing
## How was this patch tested?
manual tests
`./bin/spark-submit examples/src/main/python/ml/one_vs_rest_example.py`
Author: Zheng RuiFeng <ruifengz@foxmail.com>
Closes #12920 from zhengruifeng/ovr_pe.
Diffstat (limited to 'examples/src/main/python')
-rw-r--r-- | examples/src/main/python/ml/one_vs_rest_example.py | 68 |
1 files changed, 68 insertions, 0 deletions
diff --git a/examples/src/main/python/ml/one_vs_rest_example.py b/examples/src/main/python/ml/one_vs_rest_example.py new file mode 100644 index 0000000000..971156d0dd --- /dev/null +++ b/examples/src/main/python/ml/one_vs_rest_example.py @@ -0,0 +1,68 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function + +# $example on$ +from pyspark.ml.classification import LogisticRegression, OneVsRest +from pyspark.ml.evaluation import MulticlassClassificationEvaluator +# $example off$ +from pyspark.sql import SparkSession + +""" +An example of Multiclass to Binary Reduction with One Vs Rest, +using Logistic Regression as the base classifier. +Run with: + bin/spark-submit examples/src/main/python/ml/one_vs_rest_example.py +""" + + +if __name__ == "__main__": + spark = SparkSession \ + .builder \ + .appName("PythonOneVsRestExample") \ + .getOrCreate() + + # $example on$ + # load data file. + inputData = spark.read.format("libsvm") \ + .load("data/mllib/sample_multiclass_classification_data.txt") + + # generate the train/test split. + (train, test) = inputData.randomSplit([0.8, 0.2]) + + # instantiate the base classifier. + lr = LogisticRegression(maxIter=10, tol=1E-6, fitIntercept=True) + + # instantiate the One Vs Rest Classifier. + ovr = OneVsRest(classifier=lr) + + # train the multiclass model. + ovrModel = ovr.fit(train) + + # score the model on test data. + predictions = ovrModel.transform(test) + + # obtain evaluator. + evaluator = MulticlassClassificationEvaluator(metricName="precision") + + # compute the classification error on test data. + precision = evaluator.evaluate(predictions) + print("Test Error : " + str(1 - precision)) + # $example off$ + + spark.stop() |