[SPARK-8171][WEB UI] Javascript based infinite scrolling for the log page

Updated the log page by replacing the current pagination with a javascript-based infinite scroll solution

Author: Alex Bozarth <ajbozart@us.ibm.com>

Closes #10910 from ajbozarth/spark8171.

5 files changed, 175 insertions, 44 deletions

diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js
new file mode 100644
index 0000000000..1782b4f209
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var baseParams;
+
+var curLogLength;
+var startByte;
+var endByte;
+var totalLogLength;
+
+var byteLength;
+
+function setLogScroll(oldHeight) {
+  var logContent = $(".log-content");
+  logContent.scrollTop(logContent[0].scrollHeight - oldHeight);
+}
+
+function tailLog() {
+  var logContent = $(".log-content");
+  logContent.scrollTop(logContent[0].scrollHeight);
+}
+
+function setLogData() {
+  $('#log-data').html("Showing " + curLogLength + " Bytes: " + startByte
+    + " - " + endByte + " of " + totalLogLength);
+}
+
+function disableMoreButton() {
+  var moreBtn = $(".log-more-btn");
+  moreBtn.attr("disabled", "disabled");
+  moreBtn.html("Top of Log");
+}
+
+function noNewAlert() {
+  var alert = $(".no-new-alert");
+  alert.css("display", "block");
+  window.setTimeout(function () {alert.css("display", "none");}, 4000);
+}
+
+function loadMore() {
+  var offset = Math.max(startByte - byteLength, 0);
+  var moreByteLength = Math.min(byteLength, startByte);
+
+  $.ajax({
+    type: "GET",
+    url: "/log" + baseParams + "&offset=" + offset + "&byteLength=" + moreByteLength,
+    success: function (data) {
+      var oldHeight = $(".log-content")[0].scrollHeight;
+      var newlineIndex = data.indexOf('\n');
+      var dataInfo = data.substring(0, newlineIndex).match(/\d+/g);
+      var retStartByte = dataInfo[0];
+      var retLogLength = dataInfo[2];
+
+      var cleanData = data.substring(newlineIndex + 1);
+      if (retStartByte == 0) {
+        disableMoreButton();
+      }
+      $("pre", ".log-content").prepend(cleanData);
+
+      curLogLength = curLogLength + (startByte - retStartByte);
+      startByte = retStartByte;
+      totalLogLength = retLogLength;
+      setLogScroll(oldHeight);
+      setLogData();
+    }
+  });
+}
+
+function loadNew() {
+  $.ajax({
+    type: "GET",
+    url: "/log" + baseParams + "&byteLength=0",
+    success: function (data) {
+      var dataInfo = data.substring(0, data.indexOf('\n')).match(/\d+/g);
+      var newDataLen = dataInfo[2] - totalLogLength;
+      if (newDataLen != 0) {
+        $.ajax({
+          type: "GET",
+          url: "/log" + baseParams + "&byteLength=" + newDataLen,
+          success: function (data) {
+            var newlineIndex = data.indexOf('\n');
+            var dataInfo = data.substring(0, newlineIndex).match(/\d+/g);
+            var retStartByte = dataInfo[0];
+            var retEndByte = dataInfo[1];
+            var retLogLength = dataInfo[2];
+
+            var cleanData = data.substring(newlineIndex + 1);
+            $("pre", ".log-content").append(cleanData);
+
+            curLogLength = curLogLength + (retEndByte - retStartByte);
+            endByte = retEndByte;
+            totalLogLength = retLogLength;
+            tailLog();
+            setLogData();
+          }
+        });
+      } else {
+        noNewAlert();
+      }
+    }
+  });
+}
+
+function initLogPage(params, logLen, start, end, totLogLen, defaultLen) {
+  baseParams = params;
+  curLogLength = logLen;
+  startByte = start;
+  endByte = end;
+  totalLogLength = totLogLen;
+  byteLength = defaultLen;
+  tailLog();
+  if (startByte == 0) {
+    disableMoreButton();
+  }
+}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css
index 47dd9162a1..595e80ab5e 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css
@@ -237,3 +237,13 @@ a.expandbutton {
   color: #333;
   text-decoration: none;
 }
+
+.log-more-btn, .log-new-btn {
+  width: 100%
+}
+
+.no-new-alert {
+  text-align: center;
+  margin: 0;
+  padding: 4px 0;
+}
\ No newline at end of file
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
index e75c0cec4a..3473c41b93 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
@@ -20,7 +20,7 @@ package org.apache.spark.deploy.worker.ui
 import java.io.File
 import javax.servlet.http.HttpServletRequest
 
-import scala.xml.Node
+import scala.xml.{Node, Unparsed}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.ui.{UIUtils, WebUIPage}
@@ -31,10 +31,9 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with
   private val worker = parent.worker
   private val workDir = new File(parent.workDir.toURI.normalize().getPath)
   private val supportedLogTypes = Set("stderr", "stdout")
+  private val defaultBytes = 100 * 1024
 
   def renderLog(request: HttpServletRequest): String = {
-    val defaultBytes = 100 * 1024
-
     val appId = Option(request.getParameter("appId"))
     val executorId = Option(request.getParameter("executorId"))
     val driverId = Option(request.getParameter("driverId"))
@@ -44,9 +43,9 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with
 
     val logDir = (appId, executorId, driverId) match {
       case (Some(a), Some(e), None) =>
-        s"${workDir.getPath}/$appId/$executorId/"
+        s"${workDir.getPath}/$a/$e/"
       case (None, None, Some(d)) =>
-        s"${workDir.getPath}/$driverId/"
+        s"${workDir.getPath}/$d/"
       case _ =>
         throw new Exception("Request must specify either application or driver identifiers")
     }
@@ -57,7 +56,6 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with
   }
 
   def render(request: HttpServletRequest): Seq[Node] = {
-    val defaultBytes = 100 * 1024
     val appId = Option(request.getParameter("appId"))
     val executorId = Option(request.getParameter("executorId"))
     val driverId = Option(request.getParameter("driverId"))
@@ -76,49 +74,44 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with
 
     val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength)
     val linkToMaster = <p><a href={worker.activeMasterWebUiUrl}>Back to Master</a></p>
-    val range = <span>Bytes {startByte.toString} - {endByte.toString} of {logLength}</span>
-
-    val backButton =
-      if (startByte > 0) {
-        <a href={"?%s&logType=%s&offset=%s&byteLength=%s"
-          .format(params, logType, math.max(startByte - byteLength, 0), byteLength)}>
-          <button type="button" class="btn btn-default">
-            Previous {Utils.bytesToString(math.min(byteLength, startByte))}
-          </button>
-        </a>
-      } else {
-        <button type="button" class="btn btn-default" disabled="disabled">
-          Previous 0 B
-        </button>
-      }
+    val curLogLength = endByte - startByte
+    val range =
+      <span id="log-data">
+        Showing {curLogLength} Bytes: {startByte.toString} - {endByte.toString} of {logLength}
+      </span>
+
+    val moreButton =
+      <button type="button" onclick={"loadMore()"} class="log-more-btn btn btn-default">
+        Load More
+      </button>
+
+    val newButton =
+      <button type="button" onclick={"loadNew()"} class="log-new-btn btn btn-default">
+        Load New
+      </button>
+
+    val alert =
+      <div class="no-new-alert alert alert-info" style="display: none;">
+        End of Log
+      </div>
 
-    val nextButton =
-      if (endByte < logLength) {
-        <a href={"?%s&logType=%s&offset=%s&byteLength=%s".
-          format(params, logType, endByte, byteLength)}>
-          <button type="button" class="btn btn-default">
-            Next {Utils.bytesToString(math.min(byteLength, logLength - endByte))}
-          </button>
-        </a>
-      } else {
-        <button type="button" class="btn btn-default" disabled="disabled">
-          Next 0 B
-        </button>
-      }
+    val logParams = "?%s&logType=%s".format(params, logType)
+    val jsOnload = "window.onload = " +
+      s"initLogPage('$logParams', $curLogLength, $startByte, $endByte, $logLength, $byteLength);"
 
     val content =
       <div>
         {linkToMaster}
-        <div>
-          <div style="float:left; margin-right:10px">{backButton}</div>
-          <div style="float:left;">{range}</div>
-          <div style="float:right; margin-left:10px">{nextButton}</div>
-        </div>
-        <br />
-        <div style="height:500px; overflow:auto; padding:5px;">
+        {range}
+        <div class="log-content" style="height:80vh; overflow:auto; padding:5px;">
+          <div>{moreButton}</div>
           <pre>{logText}</pre>
+          {alert}
+          <div>{newButton}</div>
         </div>
+        <script>{Unparsed(jsOnload)}</script>
       </div>
+
     UIUtils.basicSparkPage(content, logType + " log page for " + pageName)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 119165f724..db24f0319b 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -84,9 +84,7 @@ private[spark] object JettyUtils extends Logging {
             val result = servletParams.responder(request)
             response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate")
             response.setHeader("X-Frame-Options", xFrameOptionsValue)
-            // scalastyle:off println
-            response.getWriter.println(servletParams.extractFn(result))
-            // scalastyle:on println
+            response.getWriter.print(servletParams.extractFn(result))
           } else {
             response.setStatus(HttpServletResponse.SC_UNAUTHORIZED)
             response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate")
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 28d277df4a..6241593bba 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -168,6 +168,7 @@ private[spark] object UIUtils extends Logging {
     <script src={prependBaseUri("/static/table.js")}></script>
     <script src={prependBaseUri("/static/additional-metrics.js")}></script>
     <script src={prependBaseUri("/static/timeline-view.js")}></script>
+    <script src={prependBaseUri("/static/log-view.js")}></script>
   }
 
   def vizHeaderNodes: Seq[Node] = {