diff options
| author | Andrew Or <andrewor14@gmail.com> | 2014-06-27 15:23:25 -0700 |
|---|---|---|
| committer | Patrick Wendell <pwendell@gmail.com> | 2014-06-27 15:23:34 -0700 |
| commit | 2a49a8d038ca9e600926907a955d300b43c61a1a (patch) | |
| tree | 7cb2ae6cf1a4a181171a5f50c8154ac74636e4f6 /core | |
| parent | fb61928cfb16be79effcdb48ad33fcb97d6cc8c8 (diff) | |
| download | spark-2a49a8d038ca9e600926907a955d300b43c61a1a.tar.gz spark-2a49a8d038ca9e600926907a955d300b43c61a1a.tar.bz2 spark-2a49a8d038ca9e600926907a955d300b43c61a1a.zip | |
[SPARK-2307] SparkUI - storage tab displays incorrect RDDs
The issue here is that the `StorageTab` listens for updates from the `StorageStatusListener`, but when a block is kicked out of the cache, `StorageStatusListener` removes it from its list. Thus, there is no way for the `StorageTab` to know whether a block has been dropped.
This issue was introduced in #1080, which was itself a bug fix. Here we revert that PR and offer a different fix for the original bug (SPARK-2144).
Author: Andrew Or <andrewor14@gmail.com>
Closes #1249 from andrewor14/storage-ui-fix and squashes the following commits:
af019ce [Andrew Or] Fix SPARK-2307
(cherry picked from commit 21e0f77b6321590ed86223a60cdb8ae08ea4057f)
Signed-off-by: Patrick Wendell <pwendell@gmail.com>
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala | 6 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala | 5 |
2 files changed, 5 insertions, 6 deletions
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala index c694fc8c34..a6e6627d54 100644 --- a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala +++ b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala @@ -37,11 +37,7 @@ class StorageStatusListener extends SparkListener { val filteredStatus = storageStatusList.find(_.blockManagerId.executorId == execId) filteredStatus.foreach { storageStatus => updatedBlocks.foreach { case (blockId, updatedStatus) => - if (updatedStatus.storageLevel == StorageLevel.NONE) { - storageStatus.blocks.remove(blockId) - } else { - storageStatus.blocks(blockId) = updatedStatus - } + storageStatus.blocks(blockId) = updatedStatus } } } diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 2d8c3b949c..6cfc46c7e7 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -21,6 +21,7 @@ import javax.servlet.http.HttpServletRequest import scala.xml.Node +import org.apache.spark.storage.StorageLevel import org.apache.spark.ui.{WebUIPage, UIUtils} import org.apache.spark.util.Utils @@ -107,7 +108,9 @@ private[ui] class ExecutorsPage(parent: ExecutorsTab) extends WebUIPage("") { val status = listener.storageStatusList(statusId) val execId = status.blockManagerId.executorId val hostPort = status.blockManagerId.hostPort - val rddBlocks = status.blocks.size + val rddBlocks = status.blocks.count { case (_, blockStatus) => + blockStatus.storageLevel != StorageLevel.NONE + } val memUsed = status.memUsed val maxMem = status.maxMem val diskUsed = status.diskUsed |