diff options
author | Patrick Wendell <pwendell@gmail.com> | 2014-01-18 16:29:23 -0800 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2014-01-18 16:29:23 -0800 |
commit | fe8a3546f40394466a41fc750cb60f6fc73d8bbb (patch) | |
tree | 790a7eb14b3ea2ff04d3d2d357d391eacc47f411 | |
parent | 73dfd42fba5e526cc57e2a2ed78be323b63cb8fa (diff) | |
parent | e91ad3f164b64e727f41ced6ae20d70ca4c92521 (diff) | |
download | spark-fe8a3546f40394466a41fc750cb60f6fc73d8bbb.tar.gz spark-fe8a3546f40394466a41fc750cb60f6fc73d8bbb.tar.bz2 spark-fe8a3546f40394466a41fc750cb60f6fc73d8bbb.zip |
Merge pull request #459 from srowen/UpdaterL2Regularization
Correct L2 regularized weight update with canonical form
Per thread on the user@ mailing list, and comments from Ameet, I believe the weight update for L2 regularization needs to be corrected. See http://mail-archives.apache.org/mod_mbox/spark-user/201401.mbox/%3CCAH3_EVMetuQuhj3__NdUniDLc4P-FMmmrmxw9TS14or8nT4BNQ%40mail.gmail.com%3E
-rw-r--r-- | mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala index 4c51f4f881..37124f261e 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala @@ -86,13 +86,17 @@ class L1Updater extends Updater { /** * Updater that adjusts the learning rate and performs L2 regularization + * + * See, for example, explanation of gradient and loss with L2 regularization on slide 21-22 + * of <a href="http://people.cs.umass.edu/~sheldon/teaching/2012fa/ml/files/lec7-annotated.pdf"> + * these slides</a>. */ class SquaredL2Updater extends Updater { override def compute(weightsOld: DoubleMatrix, gradient: DoubleMatrix, stepSize: Double, iter: Int, regParam: Double): (DoubleMatrix, Double) = { val thisIterStepSize = stepSize / math.sqrt(iter) val normGradient = gradient.mul(thisIterStepSize) - val newWeights = weightsOld.sub(normGradient).div(2.0 * thisIterStepSize * regParam + 1.0) + val newWeights = weightsOld.mul(1.0 - 2.0 * thisIterStepSize * regParam).sub(normGradient) (newWeights, pow(newWeights.norm2, 2.0) * regParam) } } |