diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-02-25 10:28:08 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-02-25 10:28:08 -0800 |
commit | 6ae9a22c3e15b3a4e2b0445cb89d855091746a18 (patch) | |
tree | dc977b2d75f3e9ee819486477c105ed3f65a9ea0 | |
parent | d6e6abece306008c50410807669596d73d6d6738 (diff) | |
download | spark-6ae9a22c3e15b3a4e2b0445cb89d855091746a18.tar.gz spark-6ae9a22c3e15b3a4e2b0445cb89d855091746a18.tar.bz2 spark-6ae9a22c3e15b3a4e2b0445cb89d855091746a18.zip |
Get spark.default.paralellism on each call to defaultPartitioner,
instead of only once, in case the user changes it across Spark uses
-rw-r--r-- | core/src/main/scala/spark/Partitioner.scala | 5 |
1 files changed, 1 insertions, 4 deletions
diff --git a/core/src/main/scala/spark/Partitioner.scala b/core/src/main/scala/spark/Partitioner.scala index eec0e8dd79..6f8cd17c88 100644 --- a/core/src/main/scala/spark/Partitioner.scala +++ b/core/src/main/scala/spark/Partitioner.scala @@ -10,9 +10,6 @@ abstract class Partitioner extends Serializable { } object Partitioner { - - private val useDefaultParallelism = System.getProperty("spark.default.parallelism") != null - /** * Choose a partitioner to use for a cogroup-like operation between a number of RDDs. * @@ -33,7 +30,7 @@ object Partitioner { for (r <- bySize if r.partitioner != None) { return r.partitioner.get } - if (useDefaultParallelism) { + if (System.getProperty("spark.default.parallelism") != null) { return new HashPartitioner(rdd.context.defaultParallelism) } else { return new HashPartitioner(bySize.head.partitions.size) |