diff options
author | Aaron Davidson <aaron@databricks.com> | 2013-12-31 17:42:30 -0800 |
---|---|---|
committer | Aaron Davidson <aaron@databricks.com> | 2013-12-31 17:44:15 -0800 |
commit | 08302b113a5db773e3b8d7cfea1ab1d2b8d3695b (patch) | |
tree | a808797cc8cdd44181327ae0f8926050e8dde41c /core/src/main | |
parent | 8bbe08b21ee6e48b5ba1e2c2a8b1c7eacde9603a (diff) | |
download | spark-08302b113a5db773e3b8d7cfea1ab1d2b8d3695b.tar.gz spark-08302b113a5db773e3b8d7cfea1ab1d2b8d3695b.tar.bz2 spark-08302b113a5db773e3b8d7cfea1ab1d2b8d3695b.zip |
Rename IntermediateBlockId to TempBlockId
Diffstat (limited to 'core/src/main')
4 files changed, 9 insertions, 9 deletions
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockId.scala b/core/src/main/scala/org/apache/spark/storage/BlockId.scala index c5dacf3fd2..bcc3101485 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockId.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockId.scala @@ -68,9 +68,9 @@ private[spark] case class StreamBlockId(streamId: Int, uniqueId: Long) extends B def name = "input-" + streamId + "-" + uniqueId } -/** Block associated with intermediate (temporary) data managed as blocks. */ -private[spark] case class IntermediateBlockId(id: String) extends BlockId { - def name = "intermediate_" + id +/** Block associated with temporary data managed as blocks. */ +private[spark] case class TempBlockId(id: String) extends BlockId { + def name = "temp_" + id } // Intended only for testing purposes @@ -85,7 +85,7 @@ private[spark] object BlockId { val BROADCAST_HELPER = "broadcast_([0-9]+)_([A-Za-z0-9]+)".r val TASKRESULT = "taskresult_([0-9]+)".r val STREAM = "input-([0-9]+)-([0-9]+)".r - val INTERMEDIATE = "intermediate_(.*)".r + val TEMP = "temp_(.*)".r val TEST = "test_(.*)".r /** Converts a BlockId "name" String back into a BlockId. */ diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index 58320f254a..32da458a6f 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -91,10 +91,10 @@ private[spark] class DiskBlockManager(shuffleManager: ShuffleBlockManager, rootD def getFile(blockId: BlockId): File = getFile(blockId.name) /** Produces a unique block id and File suitable for intermediate results. */ - def createIntermediateBlock: (IntermediateBlockId, File) = { - var blockId = new IntermediateBlockId(UUID.randomUUID().toString) + def createTempBlock(): (TempBlockId, File) = { + var blockId = new TempBlockId(UUID.randomUUID().toString) while (getFile(blockId).exists()) { - blockId = new IntermediateBlockId(UUID.randomUUID().toString) + blockId = new TempBlockId(UUID.randomUUID().toString) } (blockId, getFile(blockId)) } diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala index 492b4fc7c6..96f6bb3516 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala @@ -138,7 +138,7 @@ private[spark] class SpillableAppendOnlyMap[K, V, G: ClassTag, C: ClassTag]( spillCount += 1 logWarning(s"In-memory KV map exceeded threshold of $memoryThresholdMB MB!") logWarning(s"Spilling to disk ($spillCount time"+(if (spillCount > 1) "s" else "")+" so far)") - val (blockId, file) = diskBlockManager.createIntermediateBlock + val (blockId, file) = diskBlockManager.createTempBlock() val writer = new DiskBlockObjectWriter(blockId, file, serializer, fileBufferSize, identity) try { val it = currentMap.destructiveSortedIterator(comparator) diff --git a/core/src/main/scala/org/apache/spark/util/collection/SizeTrackingAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/SizeTrackingAppendOnlyMap.scala index e6b6103d96..204330dad4 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/SizeTrackingAppendOnlyMap.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/SizeTrackingAppendOnlyMap.scala @@ -96,6 +96,6 @@ private[spark] class SizeTrackingAppendOnlyMap[K, V] extends AppendOnlyMap[K, V] } } -object SizeTrackingAppendOnlyMap { +private object SizeTrackingAppendOnlyMap { case class Sample(size: Long, numUpdates: Long) } |