aboutsummaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
authorKay Ousterhout <kayousterhout@gmail.com>2014-11-24 18:03:10 -0800
committerKay Ousterhout <kayousterhout@gmail.com>2014-11-24 18:03:10 -0800
commitd24d5bf064572a2319627736b1fbf112b4a78edf (patch)
treebb1f4d0663d53be7e5ec054d0e99aab2af00a341 /core
parent6cf507685efd01df77d663145ae08e48c7f92948 (diff)
downloadspark-d24d5bf064572a2319627736b1fbf112b4a78edf.tar.gz
spark-d24d5bf064572a2319627736b1fbf112b4a78edf.tar.bz2
spark-d24d5bf064572a2319627736b1fbf112b4a78edf.zip
[SPARK-4266] [Web-UI] Reduce stage page load time.
The commit changes the java script used to show/hide additional metrics in order to reduce page load time. SPARK-4016 significantly increased page load time for the stage page when stages had a lot (thousands or tens of thousands) of tasks, due to the additional Javascript to hide some metrics by default and stripe the tables. This commit reduces page load time in two ways: (1) Now, all of the metrics that are hidden by default are hidden by setting "display: none;" using CSS for the page, rather than hiding them using javascript after the page loads. Without this change, for stages with thousands of tasks, there was a few second delay after page load, where first the additional metrics were shown, and then after a delay were hidden once the relevant JS finished running. (2) CSS is used to stripe all of the tables except for the summary table. The summary table needs javascript to do the striping because some rows are hidden, but the javascript striping is slower, which again resulted in a delay when it was used for the task table (where for a few seconds after page load, all of the rows in the task table would be white, while the browser finished running the JS to stripe the table). cc pwendell This change is intended to be backported to 1.2 to avoid a regression in UI performance when users run large jobs. Author: Kay Ousterhout <kayousterhout@gmail.com> Closes #3328 from kayousterhout/SPARK-4266 and squashes the following commits: f964091 [Kay Ousterhout] [SPARK-4266] [Web-UI] Reduce stage page load time.
Diffstat (limited to 'core')
-rw-r--r--core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js9
-rw-r--r--core/src/main/resources/org/apache/spark/ui/static/table.js24
-rw-r--r--core/src/main/resources/org/apache/spark/ui/static/webui.css6
-rw-r--r--core/src/main/scala/org/apache/spark/ui/UIUtils.scala11
-rw-r--r--core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala2
-rw-r--r--core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala2
-rw-r--r--core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala9
-rw-r--r--core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala3
8 files changed, 39 insertions, 27 deletions
diff --git a/core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js b/core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js
index d33c5c769d..14ba37d7c9 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js
@@ -28,17 +28,12 @@ $(function() {
$(this).find('.expand-additional-metrics-arrow').toggleClass('arrow-closed');
});
- $("input:checkbox:not(:checked)").each(function() {
- var column = "table ." + $(this).attr("name");
- $(column).hide();
- });
- // Stripe table rows after rows have been hidden to ensure correct striping.
- stripeTables();
+ stripeSummaryTable();
$("input:checkbox").click(function() {
var column = "table ." + $(this).attr("name");
$(column).toggle();
- stripeTables();
+ stripeSummaryTable();
});
$("#select-all-metrics").click(function() {
diff --git a/core/src/main/resources/org/apache/spark/ui/static/table.js b/core/src/main/resources/org/apache/spark/ui/static/table.js
index 6bb03015ab..656147e40d 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/table.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/table.js
@@ -15,16 +15,18 @@
* limitations under the License.
*/
-/* Adds background colors to stripe table rows. This is necessary (instead of using css or the
- * table striping provided by bootstrap) to appropriately stripe tables with hidden rows. */
-function stripeTables() {
- $("table.table-striped-custom").each(function() {
- $(this).find("tr:not(:hidden)").each(function (index) {
- if (index % 2 == 1) {
- $(this).css("background-color", "#f9f9f9");
- } else {
- $(this).css("background-color", "#ffffff");
- }
- });
+/* Adds background colors to stripe table rows in the summary table (on the stage page). This is
+ * necessary (instead of using css or the table striping provided by bootstrap) because the summary
+ * table has hidden rows.
+ *
+ * An ID selector (rather than a class selector) is used to ensure this runs quickly even on pages
+ * with thousands of task rows (ID selectors are much faster than class selectors). */
+function stripeSummaryTable() {
+ $("#task-summary-table").find("tr:not(:hidden)").each(function (index) {
+ if (index % 2 == 1) {
+ $(this).css("background-color", "#f9f9f9");
+ } else {
+ $(this).css("background-color", "#ffffff");
+ }
});
}
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css
index db57712c83..cdf85bfbf3 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css
@@ -168,3 +168,9 @@ span.additional-metric-title {
border-left: 5px solid black;
display: inline-block;
}
+
+/* Hide all additional metrics by default. This is done here rather than using JavaScript to
+ * avoid slow page loads for stage pages with large numbers (e.g., thousands) of tasks. */
+.scheduler_delay, .gc_time, .deserialization_time, .serialization_time, .getting_result_time {
+ display: none;
+}
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 0c418beaf7..09079bbd43 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -26,7 +26,8 @@ import org.apache.spark.Logging
/** Utility functions for generating XML pages with spark content. */
private[spark] object UIUtils extends Logging {
- val TABLE_CLASS = "table table-bordered table-striped-custom table-condensed sortable"
+ val TABLE_CLASS_NOT_STRIPED = "table table-bordered table-condensed sortable"
+ val TABLE_CLASS_STRIPED = TABLE_CLASS_NOT_STRIPED + " table-striped"
// SimpleDateFormat is not thread-safe. Don't expose it to avoid improper use.
private val dateFormat = new ThreadLocal[SimpleDateFormat]() {
@@ -248,12 +249,10 @@ private[spark] object UIUtils extends Logging {
data: Iterable[T],
fixedWidth: Boolean = false,
id: Option[String] = None,
- headerClasses: Seq[String] = Seq.empty): Seq[Node] = {
+ headerClasses: Seq[String] = Seq.empty,
+ stripeRowsWithCss: Boolean = true): Seq[Node] = {
- var listingTableClass = TABLE_CLASS
- if (fixedWidth) {
- listingTableClass += " table-fixed"
- }
+ val listingTableClass = if (stripeRowsWithCss) TABLE_CLASS_STRIPED else TABLE_CLASS_NOT_STRIPED
val colWidth = 100.toDouble / headers.size
val colWidthAttr = if (fixedWidth) colWidth + "%" else ""
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
index 71b59b1d07..363cb96de7 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
@@ -57,7 +57,7 @@ private[ui] class ExecutorsPage(
val execInfoSorted = execInfo.sortBy(_.id)
val execTable =
- <table class={UIUtils.TABLE_CLASS}>
+ <table class={UIUtils.TABLE_CLASS_STRIPED}>
<thead>
<th>Executor ID</th>
<th>Address</th>
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
index 35bbe8b4f9..9836d11a6d 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
@@ -36,7 +36,7 @@ private[ui] class ExecutorTable(stageId: Int, stageAttemptId: Int, parent: Stage
/** Special table which merges two header cells. */
private def executorTable[T](): Seq[Node] = {
- <table class={UIUtils.TABLE_CLASS}>
+ <table class={UIUtils.TABLE_CLASS_STRIPED}>
<thead>
<th>Executor ID</th>
<th>Address</th>
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 40e05f86b6..bfa54f8492 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -322,8 +322,15 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val quantileHeaders = Seq("Metric", "Min", "25th percentile",
"Median", "75th percentile", "Max")
+ // The summary table does not use CSS to stripe rows, which doesn't work with hidden
+ // rows (instead, JavaScript in table.js is used to stripe the non-hidden rows).
Some(UIUtils.listingTable(
- quantileHeaders, identity[Seq[Node]], listings, fixedWidth = true))
+ quantileHeaders,
+ identity[Seq[Node]],
+ listings,
+ fixedWidth = true,
+ id = Some("task-summary-table"),
+ stripeRowsWithCss = false))
}
val executorTable = new ExecutorTable(stageId, stageAttemptId, parent)
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala b/core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala
index eb371bd0ea..ca942c4051 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala
@@ -20,6 +20,9 @@ package org.apache.spark.ui.jobs
/**
* Names of the CSS classes corresponding to each type of task detail. Used to allow users
* to optionally show/hide columns.
+ *
+ * If new optional metrics are added here, they should also be added to the end of webui.css
+ * to have the style set to "display: none;" by default.
*/
private object TaskDetailsClassNames {
val SCHEDULER_DELAY = "scheduler_delay"