From 4432568aac1d4a44fa1a7c3469f095eb7a6ce945 Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Mon, 19 Jan 2015 10:10:15 -0800 Subject: [SPARK-5282][mllib]: RowMatrix easily gets int overflow in the memory size warning JIRA: https://issues.apache.org/jira/browse/SPARK-5282 fix the possible int overflow in the memory computation warning Author: Yuhao Yang Closes #4069 from hhbyyh/addscStop and squashes the following commits: e54e5c8 [Yuhao Yang] change to MB based number 7afac23 [Yuhao Yang] 5282: fix int overflow in the warning --- .../scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mllib') diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala index d5abba6a4b..02075edbab 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala @@ -131,8 +131,8 @@ class RowMatrix( throw new IllegalArgumentException(s"Argument with more than 65535 cols: $cols") } if (cols > 10000) { - val mem = cols * cols * 8 - logWarning(s"$cols columns will require at least $mem bytes of memory!") + val memMB = (cols.toLong * cols) / 125000 + logWarning(s"$cols columns will require at least $memMB megabytes of memory!") } } -- cgit v1.2.3