Clean commit of log paging

3 files changed, 75 insertions, 2 deletions

diff --git a/core/src/main/scala/spark/deploy/master/ui/ApplicationPage.scala b/core/src/main/scala/spark/deploy/master/ui/ApplicationPage.scala
index 33a16b5d84..ea88421532 100644
--- a/core/src/main/scala/spark/deploy/master/ui/ApplicationPage.scala
+++ b/core/src/main/scala/spark/deploy/master/ui/ApplicationPage.scala
@@ -38,7 +38,7 @@ private[spark] class ApplicationPage(parent: MasterWebUI) {
       state.completedApps.find(_.id == appId).getOrElse(null)
     })
 
-    val executorHeaders = Seq("ExecutorID", "Worker", "Cores", "Memory", "State", "Logs")
+    val executorHeaders = Seq("ExecutorID", "Worker", "Cores", "Memory", "State", "Logs", "Log Pages")
     val executors = app.executors.values.toSeq
     val executorTable = UIUtils.listingTable(executorHeaders, executorRow, executors)
 
@@ -95,6 +95,12 @@ private[spark] class ApplicationPage(parent: MasterWebUI) {
         <a href={"%s/log?appId=%s&executorId=%s&logType=stderr"
           .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stderr</a>
       </td>
+      <td>
+        <a href={"%s/logPage?appId=%s&executorId=%s&logType=stdout"
+          .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stdout</a>
+        <a href={"%s/logPage?appId=%s&executorId=%s&logType=stderr"
+          .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stderr</a>
+      </td>
     </tr>
   }
 }
diff --git a/core/src/main/scala/spark/deploy/worker/ui/IndexPage.scala b/core/src/main/scala/spark/deploy/worker/ui/IndexPage.scala
index e466129c1a..7cf98b473e 100644
--- a/core/src/main/scala/spark/deploy/worker/ui/IndexPage.scala
+++ b/core/src/main/scala/spark/deploy/worker/ui/IndexPage.scala
@@ -29,7 +29,7 @@ private[spark] class IndexPage(parent: WorkerWebUI) {
     val stateFuture = (worker ? RequestWorkerState)(timeout).mapTo[WorkerState]
     val workerState = Await.result(stateFuture, 30 seconds)
 
-    val executorHeaders = Seq("ExecutorID", "Cores", "Memory", "Job Details", "Logs")
+    val executorHeaders = Seq("ExecutorID", "Cores", "Memory", "Job Details", "Logs", "Log Pages")
     val runningExecutorTable =
       UIUtils.listingTable(executorHeaders, executorRow, workerState.executors)
     val finishedExecutorTable =
@@ -93,6 +93,12 @@ private[spark] class IndexPage(parent: WorkerWebUI) {
         <a href={"log?appId=%s&executorId=%s&logType=stderr"
           .format(executor.appId, executor.execId)}>stderr</a>
       </td>
+      <td>
+	 <a href={"logPage?appId=%s&executorId=%s&logType=stdout&offset=0&lineLength=20"
+          .format(executor.appId, executor.execId)}>stdout-page</a>
+	 <a href={"logPage?appId=%s&executorId=%s&logType=stderr&offset=0&lineLength=20"
+          .format(executor.appId, executor.execId)}>stderr-page</a>
+      </td> 
     </tr>
   }
 
diff --git a/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala b/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala
index 16564d5619..5b0c785b00 100644
--- a/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala
+++ b/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala
@@ -13,6 +13,10 @@ import spark.{Utils, Logging}
 import spark.ui.JettyUtils
 import spark.ui.JettyUtils._
 
+import scala.xml._
+import spark.ui.UIUtils
+import scala.io.Source._
+
 /**
  * Web UI server for the standalone worker.
  */
@@ -33,6 +37,7 @@ class WorkerWebUI(val worker: ActorRef, val workDir: File, requestedPort: Option
   val handlers = Array[(String, Handler)](
     ("/static", createStaticHandler(WorkerWebUI.STATIC_RESOURCE_DIR)),
     ("/log", (request: HttpServletRequest) => log(request)),
+    ("/logPage", (request: HttpServletRequest) => logPage(request)),
     ("/json", (request: HttpServletRequest) => indexPage.renderJson(request)),
     ("*", (request: HttpServletRequest) => indexPage.render(request))
   )
@@ -65,6 +70,62 @@ class WorkerWebUI(val worker: ActorRef, val workDir: File, requestedPort: Option
     pre + Utils.lastNBytes(path, math.min(numBytes, maxBytes))
   }
 
+  def logPage(request: HttpServletRequest): Seq[scala.xml.Node] = {
+    val appId = request.getParameter("appId")
+    val executorId = request.getParameter("executorId")
+    val logType = request.getParameter("logType")
+    val getOffset = request.getParameter("offset")
+    val getLineLength = request.getParameter("lineLength")
+    val path = "%s/%s/%s/%s".format(workDir.getPath, appId, executorId, logType)
+    val source = fromFile(path)
+    val lines = source.getLines().toArray
+    val logLength = lines.length
+    val offset = {
+      if (getOffset == null) 0
+      else if (getOffset.toInt < 0) 0
+      else getOffset.toInt
+    }
+    val lineLength = {
+      if (getLineLength == null) 0
+      else getLineLength.toInt
+    }
+    val logText = "<node>" + lines.slice(offset, offset+lineLength).mkString("\n") + "</node>"
+    val logXML = XML.loadString(logText)
+    val backButton =
+      if (offset > 0) {
+        if (offset-lineLength < 0) {
+          <a href={"?appId=%s&executorId=%s&logType=%s&offset=0&lineLength=%s".format(appId, executorId, logType, lineLength)}> <button style="float:left">back</button> </a>
+        }
+        else {
+          <a href={"?appId=%s&executorId=%s&logType=%s&offset=%s&lineLength=%s".format(appId, executorId, logType, offset-lineLength, lineLength)}> <button style="float:left">back</button> </a>
+        }
+      }
+      else {
+        <button style="float:left" disabled="disabled">back</button>
+      }
+    val nextButton =
+      if (offset+lineLength < logLength) {
+        <a href={"?appId=%s&executorId=%s&logType=%s&offset=%s&lineLength=%s".format(appId, executorId, logType, offset+lineLength, lineLength)}> <button style="float:right">next</button> </a>
+      }
+      else {
+        <button style="float:right" disabled="disabled">next</button>
+      }
+    val content =
+      <html>
+        <body>
+          <hr></hr>
+          {backButton}
+          {nextButton}
+          <br></br>
+          <pre>{logXML}</pre>
+          {backButton}
+          {nextButton}
+        </body>
+      </html>
+    source.close()
+    UIUtils.basicSparkPage(content, "Log Page for " + appId)
+  }
+
   def stop() {
     server.foreach(_.stop())
   }