diff options
| author | Andrew Or <andrew@databricks.com> | 2014-11-18 20:15:00 -0800 |
|---|---|---|
| committer | Andrew Or <andrew@databricks.com> | 2014-11-18 20:15:00 -0800 |
| commit | e22a75923e508e17b924f341d02cd5cd679210ca (patch) | |
| tree | ae64508463f2cde33252b1c68bf14fc171aa0afa | |
| parent | f9739b9c886b1c207753ebf7067c09a60eff1695 (diff) | |
| download | spark-e22a75923e508e17b924f341d02cd5cd679210ca.tar.gz spark-e22a75923e508e17b924f341d02cd5cd679210ca.tar.bz2 spark-e22a75923e508e17b924f341d02cd5cd679210ca.zip | |
[SPARK-4380] Log more precise number of bytes spilled (1.1)
This is the branch-1.1 version of #3243.
Author: Andrew Or <andrew@databricks.com>
Closes #3355 from andrewor14/spill-log-bytes-1.1 and squashes the following commits:
36ec152 [Andrew Or] Log more precise representation of bytes in spilling code
| -rw-r--r-- | core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala | 5 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala | 5 |
2 files changed, 6 insertions, 4 deletions
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala index 3e09c25990..96697d2e60 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala @@ -174,8 +174,9 @@ class ExternalAppendOnlyMap[K, V, C]( private def spill(mapSize: Long): Unit = { spillCount += 1 val threadId = Thread.currentThread().getId - logInfo("Thread %d spilling in-memory map of %d MB to disk (%d time%s so far)" - .format(threadId, mapSize / (1024 * 1024), spillCount, if (spillCount > 1) "s" else "")) + logInfo("Thread %d spilling in-memory batch of %s to disk (%d times%s so far)" + .format(threadId, org.apache.spark.util.Utils.bytesToString(mapSize), + spillCount, if (spillCount > 1) "s" else "")) val (blockId, file) = diskBlockManager.createTempLocalBlock() curWriteMetrics = new ShuffleWriteMetrics() var writer = blockManager.getDiskWriter(blockId, file, serializer, fileBufferSize, diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala index 97ddd96c98..d414ce39e9 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala @@ -268,8 +268,9 @@ private[spark] class ExternalSorter[K, V, C]( spillCount += 1 val threadId = Thread.currentThread().getId - logInfo("Thread %d spilling in-memory batch of %d MB to disk (%d spill%s so far)" - .format(threadId, memorySize / (1024 * 1024), spillCount, if (spillCount > 1) "s" else "")) + logInfo("Thread %d spilling in-memory batch of %s to disk (%d spill%s so far)" + .format(threadId, org.apache.spark.util.Utils.bytesToString(memorySize), + spillCount, if (spillCount > 1) "s" else "")) if (bypassMergeSort) { spillToPartitionFiles(collection) |