diff options
| author | Wisely Chen <wiselychen@appier.com> | 2015-07-06 16:04:01 -0700 |
|---|---|---|
| committer | Andrew Or <andrew@databricks.com> | 2015-07-06 16:04:01 -0700 |
| commit | 9ff203346ca4decf2999e33bfb8c400ec75313e6 (patch) | |
| tree | dfa9afec76f4b452762d5c7607d6d32fea517814 /core | |
| parent | 132e7fca129be8f00ba429a51bcef60abb2eed6d (diff) | |
| download | spark-9ff203346ca4decf2999e33bfb8c400ec75313e6.tar.gz spark-9ff203346ca4decf2999e33bfb8c400ec75313e6.tar.bz2 spark-9ff203346ca4decf2999e33bfb8c400ec75313e6.zip | |
[SPARK-8656] [WEBUI] Fix the webUI and JSON API number is not synced
Spark standalone master web UI show "Alive Workers" total core, total used cores and "Alive workers" total memory, memory used.
But the JSON API page "http://MASTERURL:8088/json" shows "ALL workers" core, memory number.
This webUI data is not sync with the JSON API.
The proper way is to sync the number with webUI and JSON API.
Author: Wisely Chen <wiselychen@appier.com>
Closes #7038 from thegiive/SPARK-8656 and squashes the following commits:
9e54bf0 [Wisely Chen] Change variable name to camel case
2c8ea89 [Wisely Chen] Change some styling and add local variable
431d2b0 [Wisely Chen] Worker List should contain DEAD node also
8b3b8e8 [Wisely Chen] [SPARK-8656] Fix the webUI and JSON API number is not synced
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala | 9 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala | 2 |
2 files changed, 7 insertions, 4 deletions
diff --git a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala index 2954f932b4..ccffb36652 100644 --- a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala +++ b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala @@ -76,12 +76,13 @@ private[deploy] object JsonProtocol { } def writeMasterState(obj: MasterStateResponse): JObject = { + val aliveWorkers = obj.workers.filter(_.isAlive()) ("url" -> obj.uri) ~ ("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~ - ("cores" -> obj.workers.map(_.cores).sum) ~ - ("coresused" -> obj.workers.map(_.coresUsed).sum) ~ - ("memory" -> obj.workers.map(_.memory).sum) ~ - ("memoryused" -> obj.workers.map(_.memoryUsed).sum) ~ + ("cores" -> aliveWorkers.map(_.cores).sum) ~ + ("coresused" -> aliveWorkers.map(_.coresUsed).sum) ~ + ("memory" -> aliveWorkers.map(_.memory).sum) ~ + ("memoryused" -> aliveWorkers.map(_.memoryUsed).sum) ~ ("activeapps" -> obj.activeApps.toList.map(writeApplicationInfo)) ~ ("completedapps" -> obj.completedApps.toList.map(writeApplicationInfo)) ~ ("activedrivers" -> obj.activeDrivers.toList.map(writeDriverInfo)) ~ diff --git a/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala b/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala index 471811037e..f751966605 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala @@ -105,4 +105,6 @@ private[spark] class WorkerInfo( def setState(state: WorkerState.Value): Unit = { this.state = state } + + def isAlive(): Boolean = this.state == WorkerState.ALIVE } |