1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.ml.classification
import org.apache.spark.annotation.{AlphaComponent, DeveloperApi}
import org.apache.spark.ml.param.shared._
import org.apache.spark.ml.util.SchemaUtils
import org.apache.spark.mllib.linalg.{Vector, VectorUDT}
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types.{DataType, StructType}
/**
* Params for probabilistic classification.
*/
private[classification] trait ProbabilisticClassifierParams
extends ClassifierParams with HasProbabilityCol {
override protected def validateAndTransformSchema(
schema: StructType,
fitting: Boolean,
featuresDataType: DataType): StructType = {
val parentSchema = super.validateAndTransformSchema(schema, fitting, featuresDataType)
SchemaUtils.appendColumn(parentSchema, $(probabilityCol), new VectorUDT)
}
}
/**
* :: AlphaComponent ::
*
* Single-label binary or multiclass classifier which can output class conditional probabilities.
*
* @tparam FeaturesType Type of input features. E.g., [[Vector]]
* @tparam E Concrete Estimator type
* @tparam M Concrete Model type
*
* NOTE: This is currently private[spark] but will be made public later once it is stabilized.
*/
@AlphaComponent
private[spark] abstract class ProbabilisticClassifier[
FeaturesType,
E <: ProbabilisticClassifier[FeaturesType, E, M],
M <: ProbabilisticClassificationModel[FeaturesType, M]]
extends Classifier[FeaturesType, E, M] with ProbabilisticClassifierParams {
/** @group setParam */
def setProbabilityCol(value: String): E = set(probabilityCol, value).asInstanceOf[E]
}
/**
* :: AlphaComponent ::
*
* Model produced by a [[ProbabilisticClassifier]].
* Classes are indexed {0, 1, ..., numClasses - 1}.
*
* @tparam FeaturesType Type of input features. E.g., [[Vector]]
* @tparam M Concrete Model type
*
* NOTE: This is currently private[spark] but will be made public later once it is stabilized.
*/
@AlphaComponent
private[spark] abstract class ProbabilisticClassificationModel[
FeaturesType,
M <: ProbabilisticClassificationModel[FeaturesType, M]]
extends ClassificationModel[FeaturesType, M] with ProbabilisticClassifierParams {
/** @group setParam */
def setProbabilityCol(value: String): M = set(probabilityCol, value).asInstanceOf[M]
/**
* Transforms dataset by reading from [[featuresCol]], and appending new columns as specified by
* parameters:
* - predicted labels as [[predictionCol]] of type [[Double]]
* - raw predictions (confidences) as [[rawPredictionCol]] of type [[Vector]]
* - probability of each class as [[probabilityCol]] of type [[Vector]].
*
* @param dataset input dataset
* @return transformed dataset
*/
override def transform(dataset: DataFrame): DataFrame = {
// This default implementation should be overridden as needed.
// Check schema
transformSchema(dataset.schema, logging = true)
val (numColsOutput, outputData) =
ClassificationModel.transformColumnsImpl[FeaturesType](dataset, this)
// Output selected columns only.
if ($(probabilityCol) != "") {
// output probabilities
outputData.withColumn($(probabilityCol),
callUDF(predictProbabilities _, new VectorUDT, col($(featuresCol))))
} else {
if (numColsOutput == 0) {
this.logWarning(s"$uid: ProbabilisticClassificationModel.transform() was called as NOOP" +
" since no output columns were set.")
}
outputData
}
}
/**
* :: DeveloperApi ::
*
* Predict the probability of each class given the features.
* These predictions are also called class conditional probabilities.
*
* WARNING: Not all models output well-calibrated probability estimates! These probabilities
* should be treated as confidences, not precise probabilities.
*
* This internal method is used to implement [[transform()]] and output [[probabilityCol]].
*/
@DeveloperApi
protected def predictProbabilities(features: FeaturesType): Vector
}
|
