[SPARK-15951] Change Executors Page to use datatables to support sorting columns and searching

1. Create the executorspage-template.html for displaying application information in datables.
2. Added REST API endpoint "allexecutors" to be able to see all executors created for particular job.
3. The executorspage.js uses jQuery to access the data from /api/v1/applications/appid/allexecutors REST API, and use DataTable to display executors for the application. It also, generates summary of dead/live and total executors created during life of the application.
4. Similar changes applicable to Executors Page on history server for a given application.

Snapshots for how it looks like now:
<img width="938" alt="screen shot 2016-06-14 at 2 45 44 pm" src="https://cloud.githubusercontent.com/assets/6090397/16060092/ad1de03a-324b-11e6-8469-9eaa3f2548b5.png">

New Executors Page screenshot looks like this:
<img width="1436" alt="screen shot 2016-06-15 at 10 12 01 am" src="https://cloud.githubusercontent.com/assets/6090397/16085514/ee7004f0-32e1-11e6-9340-33d91e407f2b.png">

Author: Kishor Patil <kpatil@yahoo-inc.com>

Closes #13670 from kishorvpatil/execTemplates.

5 files changed, 70 insertions, 279 deletions

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
index 2fad1120cd..74f78021ed 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
@@ -43,8 +43,9 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
             {
             if (allAppsSize > 0) {
               <script src={UIUtils.prependBaseUri("/static/dataTables.rowsGroup.js")}></script> ++
-              <div id="history-summary" class="span12 pagination"></div> ++
-              <script src={UIUtils.prependBaseUri("/static/historypage.js")}> </script>
+                <div id="history-summary" class="span12 pagination"></div> ++
+                <script src={UIUtils.prependBaseUri("/static/utils.js")}></script> ++
+                <script src={UIUtils.prependBaseUri("/static/historypage.js")}></script>
             } else if (requestedIncomplete) {
               <h4>No incomplete applications found!</h4>
             } else {
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala
new file mode 100644
index 0000000000..01f2a18122
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala
@@ -0,0 +1,41 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.spark.status.api.v1
+
+import javax.ws.rs.{GET, Produces}
+import javax.ws.rs.core.MediaType
+
+import org.apache.spark.ui.SparkUI
+import org.apache.spark.ui.exec.ExecutorsPage
+
+@Produces(Array(MediaType.APPLICATION_JSON))
+private[v1] class AllExecutorListResource(ui: SparkUI) {
+
+  @GET
+  def executorList(): Seq[ExecutorSummary] = {
+    val listener = ui.executorsListener
+    listener.synchronized {
+      // The follow codes should be protected by `listener` to make sure no executors will be
+      // removed before we query their status. See SPARK-12784.
+      (0 until listener.activeStorageStatusList.size).map { statusId =>
+        ExecutorsPage.getExecInfo(listener, statusId, isActive = true)
+      } ++ (0 until listener.deadStorageStatusList.size).map { statusId =>
+        ExecutorsPage.getExecInfo(listener, statusId, isActive = false)
+      }
+    }
+  }
+}
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
index 681f295006..de927117e1 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
@@ -91,6 +91,13 @@ private[v1] class ApiRootResource extends UIRootFromServletContext {
     }
   }
 
+  @Path("applications/{appId}/allexecutors")
+  def getAllExecutors(@PathParam("appId") appId: String): AllExecutorListResource = {
+    uiRoot.withSparkUI(appId, None) { ui =>
+      new AllExecutorListResource(ui)
+    }
+  }
+
   @Path("applications/{appId}/{attemptId}/executors")
   def getExecutors(
       @PathParam("appId") appId: String,
@@ -100,6 +107,15 @@ private[v1] class ApiRootResource extends UIRootFromServletContext {
     }
   }
 
+  @Path("applications/{appId}/{attemptId}/allexecutors")
+  def getAllExecutors(
+      @PathParam("appId") appId: String,
+      @PathParam("attemptId") attemptId: String): AllExecutorListResource = {
+    uiRoot.withSparkUI(appId, Some(attemptId)) { ui =>
+      new AllExecutorListResource(ui)
+    }
+  }
+
 
   @Path("applications/{appId}/stages")
   def getStages(@PathParam("appId") appId: String): AllStagesResource = {
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 740f5e5f7f..2b6c538485 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -201,7 +201,8 @@ private[spark] object UIUtils extends Logging {
       activeTab: SparkUITab,
       refreshInterval: Option[Int] = None,
       helpText: Option[String] = None,
-      showVisualization: Boolean = false): Seq[Node] = {
+      showVisualization: Boolean = false,
+      useDataTables: Boolean = false): Seq[Node] = {
 
     val appName = activeTab.appName
     val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..."
@@ -216,6 +217,7 @@ private[spark] object UIUtils extends Logging {
       <head>
         {commonHeaderNodes}
         {if (showVisualization) vizHeaderNodes else Seq.empty}
+        {if (useDataTables) dataTablesHeaderNodes else Seq.empty}
         <title>{appName} - {title}</title>
       </head>
       <body>
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
index 67deb7b14b..287390b87b 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
@@ -20,7 +20,6 @@ package org.apache.spark.ui.exec
 import java.net.URLEncoder
 import javax.servlet.http.HttpServletRequest
 
-import scala.util.Try
 import scala.xml.Node
 
 import org.apache.spark.status.api.v1.ExecutorSummary
@@ -54,285 +53,17 @@ private[ui] class ExecutorsPage(
   // When GCTimePercent is edited change ToolTips.TASK_TIME to match
   private val GCTimePercent = 0.1
 
-  // a safe String to Int for sorting ids (converts non-numeric Strings to -1)
-  private def idStrToInt(str: String) : Int = Try(str.toInt).getOrElse(-1)
-
   def render(request: HttpServletRequest): Seq[Node] = {
-    val (activeExecutorInfo, deadExecutorInfo) = listener.synchronized {
-      // The follow codes should be protected by `listener` to make sure no executors will be
-      // removed before we query their status. See SPARK-12784.
-      val _activeExecutorInfo = {
-        for (statusId <- 0 until listener.activeStorageStatusList.size)
-          yield ExecutorsPage.getExecInfo(listener, statusId, isActive = true)
-      }
-      val _deadExecutorInfo = {
-        for (statusId <- 0 until listener.deadStorageStatusList.size)
-          yield ExecutorsPage.getExecInfo(listener, statusId, isActive = false)
-      }
-      (_activeExecutorInfo, _deadExecutorInfo)
-    }
-
-    val execInfo = activeExecutorInfo ++ deadExecutorInfo
-    implicit val idOrder = Ordering[Int].on((s: String) => idStrToInt(s)).reverse
-    val execInfoSorted = execInfo.sortBy(_.id)
-    val logsExist = execInfo.filter(_.executorLogs.nonEmpty).nonEmpty
-
-    val execTable = {
-      <table class={UIUtils.TABLE_CLASS_STRIPED_SORTABLE}>
-        <thead>
-          <th class="sorttable_numeric">Executor ID</th>
-          <th>Address</th>
-          <th>Status</th>
-          <th>RDD Blocks</th>
-          <th><span data-toggle="tooltip" title={ToolTips.STORAGE_MEMORY}>Storage Memory</span></th>
-          <th>Disk Used</th>
-          <th>Cores</th>
-          <th>Active Tasks</th>
-          <th>Failed Tasks</th>
-          <th>Complete Tasks</th>
-          <th>Total Tasks</th>
-          <th><span data-toggle="tooltip" title={ToolTips.TASK_TIME}>Task Time (GC Time)</span></th>
-          <th><span data-toggle="tooltip" title={ToolTips.INPUT}>Input</span></th>
-          <th><span data-toggle="tooltip" title={ToolTips.SHUFFLE_READ}>Shuffle Read</span></th>
-          <th>
-            <!-- Place the shuffle write tooltip on the left (rather than the default position
-              of on top) because the shuffle write column is the last column on the right side and
-              the tooltip is wider than the column, so it doesn't fit on top. -->
-            <span data-toggle="tooltip" data-placement="left" title={ToolTips.SHUFFLE_WRITE}>
-              Shuffle Write
-            </span>
-          </th>
-          {if (logsExist) <th class="sorttable_nosort">Logs</th> else Seq.empty}
-          {if (threadDumpEnabled) <th class="sorttable_nosort">Thread Dump</th> else Seq.empty}
-        </thead>
-        <tbody>
-          {execInfoSorted.map(execRow(_, logsExist))}
-        </tbody>
-      </table>
-    }
-
     val content =
-      <div class="row">
-        <div class="span12">
-          <h4>Summary</h4>
-          {execSummary(activeExecutorInfo, deadExecutorInfo)}
-        </div>
-      </div>
-      <div class = "row">
-        <div class="span12">
-          <h4>Executors</h4>
-          {execTable}
-        </div>
-      </div>;
-
-    UIUtils.headerSparkPage("Executors", content, parent)
-  }
-
-  /** Render an HTML row representing an executor */
-  private def execRow(info: ExecutorSummary, logsExist: Boolean): Seq[Node] = {
-    val maximumMemory = info.maxMemory
-    val memoryUsed = info.memoryUsed
-    val diskUsed = info.diskUsed
-    val executorStatus =
-      if (info.isActive) {
-        "Active"
-      } else {
-        "Dead"
-      }
-
-    <tr>
-      <td sorttable_customkey={idStrToInt(info.id).toString}>{info.id}</td>
-      <td>{info.hostPort}</td>
-      <td sorttable_customkey={executorStatus.toString}>
-        {executorStatus}
-      </td>
-      <td>{info.rddBlocks}</td>
-      <td sorttable_customkey={memoryUsed.toString}>
-        {Utils.bytesToString(memoryUsed)} /
-        {Utils.bytesToString(maximumMemory)}
-      </td>
-      <td sorttable_customkey={diskUsed.toString}>
-        {Utils.bytesToString(diskUsed)}
-      </td>
-      <td>{info.totalCores}</td>
-      {taskData(info.maxTasks, info.activeTasks, info.failedTasks, info.completedTasks,
-      info.totalTasks, info.totalDuration, info.totalGCTime)}
-      <td sorttable_customkey={info.totalInputBytes.toString}>
-        {Utils.bytesToString(info.totalInputBytes)}
-      </td>
-      <td sorttable_customkey={info.totalShuffleRead.toString}>
-        {Utils.bytesToString(info.totalShuffleRead)}
-      </td>
-      <td sorttable_customkey={info.totalShuffleWrite.toString}>
-        {Utils.bytesToString(info.totalShuffleWrite)}
-      </td>
-      {
-        if (logsExist) {
-          <td>
-            {
-              info.executorLogs.map { case (logName, logUrl) =>
-                <div>
-                  <a href={logUrl}>
-                    {logName}
-                  </a>
-                </div>
-              }
-            }
-          </td>
+      <div>
+        {
+        <div id="active-executors"></div> ++
+          <script src={UIUtils.prependBaseUri("/static/utils.js")}></script> ++
+          <script src={UIUtils.prependBaseUri("/static/executorspage.js")}></script>
         }
-      }
-      {
-        if (threadDumpEnabled) {
-          if (info.isActive) {
-            val encodedId = URLEncoder.encode(info.id, "UTF-8")
-            <td>
-              <a href={s"threadDump/?executorId=${encodedId}"}>Thread Dump</a>
-            </td>
-          } else {
-            <td> </td>
-          }
-        } else {
-          Seq.empty
-        }
-      }
-    </tr>
-  }
-
-  private def execSummaryRow(execInfo: Seq[ExecutorSummary], rowName: String): Seq[Node] = {
-    val maximumMemory = execInfo.map(_.maxMemory).sum
-    val memoryUsed = execInfo.map(_.memoryUsed).sum
-    val diskUsed = execInfo.map(_.diskUsed).sum
-    val totalCores = execInfo.map(_.totalCores).sum
-    val totalInputBytes = execInfo.map(_.totalInputBytes).sum
-    val totalShuffleRead = execInfo.map(_.totalShuffleRead).sum
-    val totalShuffleWrite = execInfo.map(_.totalShuffleWrite).sum
-
-    <tr>
-      <td><b>{rowName}({execInfo.size})</b></td>
-      <td>{execInfo.map(_.rddBlocks).sum}</td>
-      <td sorttable_customkey={memoryUsed.toString}>
-        {Utils.bytesToString(memoryUsed)} /
-        {Utils.bytesToString(maximumMemory)}
-      </td>
-      <td sorttable_customkey={diskUsed.toString}>
-        {Utils.bytesToString(diskUsed)}
-      </td>
-      <td>{totalCores}</td>
-      {taskData(execInfo.map(_.maxTasks).sum,
-      execInfo.map(_.activeTasks).sum,
-      execInfo.map(_.failedTasks).sum,
-      execInfo.map(_.completedTasks).sum,
-      execInfo.map(_.totalTasks).sum,
-      execInfo.map(_.totalDuration).sum,
-      execInfo.map(_.totalGCTime).sum)}
-      <td sorttable_customkey={totalInputBytes.toString}>
-        {Utils.bytesToString(totalInputBytes)}
-      </td>
-      <td sorttable_customkey={totalShuffleRead.toString}>
-        {Utils.bytesToString(totalShuffleRead)}
-      </td>
-      <td sorttable_customkey={totalShuffleWrite.toString}>
-        {Utils.bytesToString(totalShuffleWrite)}
-      </td>
-    </tr>
-  }
-
-  private def execSummary(activeExecInfo: Seq[ExecutorSummary], deadExecInfo: Seq[ExecutorSummary]):
-    Seq[Node] = {
-    val totalExecInfo = activeExecInfo ++ deadExecInfo
-    val activeRow = execSummaryRow(activeExecInfo, "Active");
-    val deadRow = execSummaryRow(deadExecInfo, "Dead");
-    val totalRow = execSummaryRow(totalExecInfo, "Total");
-
-    <table class={UIUtils.TABLE_CLASS_STRIPED}>
-      <thead>
-        <th></th>
-        <th>RDD Blocks</th>
-        <th><span data-toggle="tooltip" title={ToolTips.STORAGE_MEMORY}>Storage Memory</span></th>
-        <th>Disk Used</th>
-        <th>Cores</th>
-        <th>Active Tasks</th>
-        <th>Failed Tasks</th>
-        <th>Complete Tasks</th>
-        <th>Total Tasks</th>
-        <th><span data-toggle="tooltip" title={ToolTips.TASK_TIME}>Task Time (GC Time)</span></th>
-        <th><span data-toggle="tooltip" title={ToolTips.INPUT}>Input</span></th>
-        <th><span data-toggle="tooltip" title={ToolTips.SHUFFLE_READ}>Shuffle Read</span></th>
-        <th>
-          <span data-toggle="tooltip" data-placement="left" title={ToolTips.SHUFFLE_WRITE}>
-            Shuffle Write
-          </span>
-        </th>
-      </thead>
-      <tbody>
-        {activeRow}
-        {deadRow}
-        {totalRow}
-      </tbody>
-    </table>
-  }
-
-  private def taskData(
-      maxTasks: Int,
-      activeTasks: Int,
-      failedTasks: Int,
-      completedTasks: Int,
-      totalTasks: Int,
-      totalDuration: Long,
-      totalGCTime: Long): Seq[Node] = {
-    // Determine Color Opacity from 0.5-1
-    // activeTasks range from 0 to maxTasks
-    val activeTasksAlpha =
-      if (maxTasks > 0) {
-        (activeTasks.toDouble / maxTasks) * 0.5 + 0.5
-      } else {
-        1
-      }
-    // failedTasks range max at 10% failure, alpha max = 1
-    val failedTasksAlpha =
-      if (totalTasks > 0) {
-        math.min(10 * failedTasks.toDouble / totalTasks, 1) * 0.5 + 0.5
-      } else {
-        1
-      }
-    // totalDuration range from 0 to 50% GC time, alpha max = 1
-    val totalDurationAlpha =
-      if (totalDuration > 0) {
-        math.min(totalGCTime.toDouble / totalDuration + 0.5, 1)
-      } else {
-        1
-      }
-
-    val tableData =
-    <td style={
-      if (activeTasks > 0) {
-        "background:hsla(240, 100%, 50%, " + activeTasksAlpha + ");color:white"
-      } else {
-        ""
-      }
-      }>{activeTasks}</td>
-    <td style={
-      if (failedTasks > 0) {
-        "background:hsla(0, 100%, 50%, " + failedTasksAlpha + ");color:white"
-      } else {
-        ""
-      }
-      }>{failedTasks}</td>
-    <td>{completedTasks}</td>
-    <td>{totalTasks}</td>
-    <td sorttable_customkey={totalDuration.toString} style={
-      // Red if GC time over GCTimePercent of total time
-      if (totalGCTime > GCTimePercent * totalDuration) {
-        "background:hsla(0, 100%, 50%, " + totalDurationAlpha + ");color:white"
-      } else {
-        ""
-      }
-    }>
-      {Utils.msDurationToString(totalDuration)}
-      ({Utils.msDurationToString(totalGCTime)})
-    </td>;
+      </div>;
 
-    tableData
+    UIUtils.headerSparkPage("Executors", content, parent, useDataTables = true)
   }
 }