diff options
author | wangfei <scnbwf@yeah.net> | 2014-08-05 00:51:07 -0700 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2014-08-05 00:51:07 -0700 |
commit | 9862c614c06507aa7624208f1d7ed5bc027ca52e (patch) | |
tree | 1d355c483584db1fab31ab203481e3737f772569 | |
parent | a646a365e3beb8d0cd7e492e625ce68ee9439a07 (diff) | |
download | spark-9862c614c06507aa7624208f1d7ed5bc027ca52e.tar.gz spark-9862c614c06507aa7624208f1d7ed5bc027ca52e.tar.bz2 spark-9862c614c06507aa7624208f1d7ed5bc027ca52e.zip |
[SPARK-1779] Throw an exception if memory fractions are not between 0 and 1
Author: wangfei <scnbwf@yeah.net>
Author: wangfei <wangfei1@huawei.com>
Closes #714 from scwf/memoryFraction and squashes the following commits:
6e385b9 [wangfei] Update SparkConf.scala
da6ee59 [wangfei] add configs
829a195 [wangfei] add indent
717c0ca [wangfei] updated to make more concise
fc45476 [wangfei] validate memoryfraction in sparkconf
2e79b3d [wangfei] && => ||
43621bd [wangfei] && => ||
cf38bcf [wangfei] throw IllegalArgumentException
14d18ac [wangfei] throw IllegalArgumentException
dff1f0f [wangfei] Update BlockManager.scala
764965f [wangfei] Update ExternalAppendOnlyMap.scala
a59d76b [wangfei] Throw exception when memoryFracton is out of range
7b899c2 [wangfei] 【SPARK-1779】
-rw-r--r-- | core/src/main/scala/org/apache/spark/SparkConf.scala | 14 |
1 files changed, 14 insertions, 0 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala index 38700847c8..cce7a23d3b 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala @@ -238,6 +238,20 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging { } } + // Validate memory fractions + val memoryKeys = Seq( + "spark.storage.memoryFraction", + "spark.shuffle.memoryFraction", + "spark.shuffle.safetyFraction", + "spark.storage.unrollFraction", + "spark.storage.safetyFraction") + for (key <- memoryKeys) { + val value = getDouble(key, 0.5) + if (value > 1 || value < 0) { + throw new IllegalArgumentException("$key should be between 0 and 1 (was '$value').") + } + } + // Check for legacy configs sys.env.get("SPARK_JAVA_OPTS").foreach { value => val warning = |