diff options
author | Junyang Qian <junyangq@databricks.com> | 2016-09-03 12:26:30 -0700 |
---|---|---|
committer | Felix Cheung <felixcheung@apache.org> | 2016-09-03 12:26:30 -0700 |
commit | abb2f921036d97d8cab033838ae559eb731bf0fd (patch) | |
tree | 0bfe7656e726402591cba3def5fe77ff09526f74 /mllib/src | |
parent | c2a1576c230697f56f282b6388c79835377e0f2f (diff) | |
download | spark-abb2f921036d97d8cab033838ae559eb731bf0fd.tar.gz spark-abb2f921036d97d8cab033838ae559eb731bf0fd.tar.bz2 spark-abb2f921036d97d8cab033838ae559eb731bf0fd.zip |
[SPARK-17315][SPARKR] Kolmogorov-Smirnov test SparkR wrapper
## What changes were proposed in this pull request?
This PR tries to add Kolmogorov-Smirnov Test wrapper to SparkR. This wrapper implementation only supports one sample test against normal distribution.
## How was this patch tested?
R unit test.
Author: Junyang Qian <junyangq@databricks.com>
Closes #14881 from junyangq/SPARK-17315.
Diffstat (limited to 'mllib/src')
-rw-r--r-- | mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala | 57 |
1 files changed, 57 insertions, 0 deletions
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala new file mode 100644 index 0000000000..21531eb057 --- /dev/null +++ b/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.ml.r + +import org.apache.spark.mllib.stat.Statistics.kolmogorovSmirnovTest +import org.apache.spark.mllib.stat.test.KolmogorovSmirnovTestResult +import org.apache.spark.sql.{DataFrame, Row} + +private[r] class KSTestWrapper private ( + val testResult: KolmogorovSmirnovTestResult, + val distName: String, + val distParams: Array[Double]) { + + lazy val pValue = testResult.pValue + + lazy val statistic = testResult.statistic + + lazy val nullHypothesis = testResult.nullHypothesis + + lazy val degreesOfFreedom = testResult.degreesOfFreedom + + def summary: String = testResult.toString +} + +private[r] object KSTestWrapper { + + def test( + data: DataFrame, + featureName: String, + distName: String, + distParams: Array[Double]): KSTestWrapper = { + + val rddData = data.select(featureName).rdd.map { + case Row(feature: Double) => feature + } + + val ksTestResult = kolmogorovSmirnovTest(rddData, distName, distParams : _*) + + new KSTestWrapper(ksTestResult, distName, distParams) + } +} + |