aboutsummaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
authorzsxwing <zsxwing@gmail.com>2015-06-30 00:46:55 +0900
committerKousuke Saruta <sarutak@oss.nttdata.co.jp>2015-06-30 00:46:55 +0900
commit630bd5fd80193ab6dc6ad0e7bcc13ee0dadabd38 (patch)
treec7f1c130c950b885f027fe62329ebf6160fc4058 /core
parent660c6cec75dc165cf5d62cdc1b0951bdb93df365 (diff)
downloadspark-630bd5fd80193ab6dc6ad0e7bcc13ee0dadabd38.tar.gz
spark-630bd5fd80193ab6dc6ad0e7bcc13ee0dadabd38.tar.bz2
spark-630bd5fd80193ab6dc6ad0e7bcc13ee0dadabd38.zip
[SPARK-8702] [WEBUI] Avoid massive concating strings in Javascript
When there are massive tasks, such as `sc.parallelize(1 to 100000, 10000).count()`, the generated JS codes have a lot of string concatenations in the stage page, nearly 40 string concatenations for one task. We can generate the whole string for a task instead of execution string concatenations in the browser. Before this patch, the load time of the page is about 21 seconds. ![screen shot 2015-06-29 at 6 44 04 pm](https://cloud.githubusercontent.com/assets/1000778/8406644/eb55ed18-1e90-11e5-9ad5-50d27ad1dff1.png) After this patch, it reduces to about 17 seconds. ![screen shot 2015-06-29 at 6 47 34 pm](https://cloud.githubusercontent.com/assets/1000778/8406665/087003ca-1e91-11e5-80a8-3485aa9adafa.png) One disadvantage is that the generated JS codes become hard to read. Author: zsxwing <zsxwing@gmail.com> Closes #7082 from zsxwing/js-string and squashes the following commits: b29231d [zsxwing] Avoid massive concating strings in Javascript
Diffstat (limited to 'core')
-rw-r--r--core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala88
1 files changed, 44 insertions, 44 deletions
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index b83a49f79c..e96bf49d0d 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -572,55 +572,55 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val attempt = taskInfo.attempt
val timelineObject =
s"""
- {
- 'className': 'task task-assignment-timeline-object',
- 'group': '$executorId',
- 'content': '<div class="task-assignment-timeline-content"' +
- 'data-toggle="tooltip" data-placement="top"' +
- 'data-html="true" data-container="body"' +
- 'data-title="${s"Task " + index + " (attempt " + attempt + ")"}<br>' +
- 'Status: ${taskInfo.status}<br>' +
- 'Launch Time: ${UIUtils.formatDate(new Date(launchTime))}' +
- '${
+ |{
+ |'className': 'task task-assignment-timeline-object',
+ |'group': '$executorId',
+ |'content': '<div class="task-assignment-timeline-content"
+ |data-toggle="tooltip" data-placement="top"
+ |data-html="true" data-container="body"
+ |data-title="${s"Task " + index + " (attempt " + attempt + ")"}<br>
+ |Status: ${taskInfo.status}<br>
+ |Launch Time: ${UIUtils.formatDate(new Date(launchTime))}
+ |${
if (!taskInfo.running) {
s"""<br>Finish Time: ${UIUtils.formatDate(new Date(finishTime))}"""
} else {
""
}
- }' +
- '<br>Scheduler Delay: $schedulerDelay ms' +
- '<br>Task Deserialization Time: ${UIUtils.formatDuration(deserializationTime)}' +
- '<br>Shuffle Read Time: ${UIUtils.formatDuration(shuffleReadTime)}' +
- '<br>Executor Computing Time: ${UIUtils.formatDuration(executorComputingTime)}' +
- '<br>Shuffle Write Time: ${UIUtils.formatDuration(shuffleWriteTime)}' +
- '<br>Result Serialization Time: ${UIUtils.formatDuration(serializationTime)}' +
- '<br>Getting Result Time: ${UIUtils.formatDuration(gettingResultTime)}">' +
- '<svg class="task-assignment-timeline-duration-bar">' +
- '<rect class="scheduler-delay-proportion" ' +
- 'x="$schedulerDelayProportionPos%" y="0px" height="26px"' +
- 'width="$schedulerDelayProportion%""></rect>' +
- '<rect class="deserialization-time-proportion" '+
- 'x="$deserializationTimeProportionPos%" y="0px" height="26px"' +
- 'width="$deserializationTimeProportion%"></rect>' +
- '<rect class="shuffle-read-time-proportion" ' +
- 'x="$shuffleReadTimeProportionPos%" y="0px" height="26px"' +
- 'width="$shuffleReadTimeProportion%"></rect>' +
- '<rect class="executor-runtime-proportion" ' +
- 'x="$executorRuntimeProportionPos%" y="0px" height="26px"' +
- 'width="$executorComputingTimeProportion%"></rect>' +
- '<rect class="shuffle-write-time-proportion" ' +
- 'x="$shuffleWriteTimeProportionPos%" y="0px" height="26px"' +
- 'width="$shuffleWriteTimeProportion%"></rect>' +
- '<rect class="serialization-time-proportion" ' +
- 'x="$serializationTimeProportionPos%" y="0px" height="26px"' +
- 'width="$serializationTimeProportion%"></rect>' +
- '<rect class="getting-result-time-proportion" ' +
- 'x="$gettingResultTimeProportionPos%" y="0px" height="26px"' +
- 'width="$gettingResultTimeProportion%"></rect></svg>',
- 'start': new Date($launchTime),
- 'end': new Date($finishTime)
- }
- """
+ }
+ |<br>Scheduler Delay: $schedulerDelay ms
+ |<br>Task Deserialization Time: ${UIUtils.formatDuration(deserializationTime)}
+ |<br>Shuffle Read Time: ${UIUtils.formatDuration(shuffleReadTime)}
+ |<br>Executor Computing Time: ${UIUtils.formatDuration(executorComputingTime)}
+ |<br>Shuffle Write Time: ${UIUtils.formatDuration(shuffleWriteTime)}
+ |<br>Result Serialization Time: ${UIUtils.formatDuration(serializationTime)}
+ |<br>Getting Result Time: ${UIUtils.formatDuration(gettingResultTime)}">
+ |<svg class="task-assignment-timeline-duration-bar">
+ |<rect class="scheduler-delay-proportion"
+ |x="$schedulerDelayProportionPos%" y="0px" height="26px"
+ |width="$schedulerDelayProportion%""></rect>
+ |<rect class="deserialization-time-proportion"
+ |x="$deserializationTimeProportionPos%" y="0px" height="26px"
+ |width="$deserializationTimeProportion%"></rect>
+ |<rect class="shuffle-read-time-proportion"
+ |x="$shuffleReadTimeProportionPos%" y="0px" height="26px"
+ |width="$shuffleReadTimeProportion%"></rect>
+ |<rect class="executor-runtime-proportion"
+ |x="$executorRuntimeProportionPos%" y="0px" height="26px"
+ |width="$executorComputingTimeProportion%"></rect>
+ |<rect class="shuffle-write-time-proportion"
+ |x="$shuffleWriteTimeProportionPos%" y="0px" height="26px"
+ |width="$shuffleWriteTimeProportion%"></rect>
+ |<rect class="serialization-time-proportion"
+ |x="$serializationTimeProportionPos%" y="0px" height="26px"
+ |width="$serializationTimeProportion%"></rect>
+ |<rect class="getting-result-time-proportion"
+ |x="$gettingResultTimeProportionPos%" y="0px" height="26px"
+ |width="$gettingResultTimeProportion%"></rect></svg>',
+ |'start': new Date($launchTime),
+ |'end': new Date($finishTime)
+ |}
+ |""".stripMargin.replaceAll("\n", " ")
timelineObject
}.mkString("[", ",", "]")