diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-03-06 11:11:20 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-03-06 11:11:20 -0800 |
commit | 370b95816f0adbb1e47508b38240eef8f36367bb (patch) | |
tree | 50aac78d1bd2185d038f8b53febe239d0cc24a1b | |
parent | 021c50a8d4f00a008c7af24893fde702ea004523 (diff) | |
download | spark-370b95816f0adbb1e47508b38240eef8f36367bb.tar.gz spark-370b95816f0adbb1e47508b38240eef8f36367bb.tar.bz2 spark-370b95816f0adbb1e47508b38240eef8f36367bb.zip |
Added sampling for large arrays in SizeEstimator
-rw-r--r-- | core/src/main/scala/spark/SizeEstimator.scala | 17 |
1 files changed, 15 insertions, 2 deletions
diff --git a/core/src/main/scala/spark/SizeEstimator.scala b/core/src/main/scala/spark/SizeEstimator.scala index 12dd19d704..a3774fb055 100644 --- a/core/src/main/scala/spark/SizeEstimator.scala +++ b/core/src/main/scala/spark/SizeEstimator.scala @@ -5,6 +5,7 @@ import java.lang.reflect.Modifier import java.lang.reflect.{Array => JArray} import java.util.IdentityHashMap import java.util.concurrent.ConcurrentHashMap +import java.util.Random import scala.collection.mutable.ArrayBuffer @@ -98,8 +99,20 @@ object SizeEstimator { state.size += length * primitiveSize(elementClass) } else { state.size += length * POINTER_SIZE - for (i <- 0 until length) { - state.enqueue(JArray.get(array, i)) + if (length <= 100) { + for (i <- 0 until length) { + state.enqueue(JArray.get(array, i)) + } + } else { + // Estimate the size of a large array by sampling elements. + // TODO: Add a config setting for turning this off? + var size = 0.0 + val rand = new Random(42) + for (i <- 0 until 100) { + val elem = JArray.get(array, rand.nextInt(length)) + size += SizeEstimator.estimate(elem) + } + state.size += ((length / 100.0) * size).toLong } } } |