diff options
author | Karen Feng <karenfeng.us@gmail.com> | 2013-07-09 17:25:10 -0700 |
---|---|---|
committer | Karen Feng <karenfeng.us@gmail.com> | 2013-07-09 17:25:10 -0700 |
commit | b6072b58bf795093df0e5f8424413fcd0cce6323 (patch) | |
tree | 29c2256acf0a089110aae34c9ca4522370396008 /core/src/main | |
parent | 13fc6f248c7231501d8c53ad13641fa996e06be4 (diff) | |
download | spark-b6072b58bf795093df0e5f8424413fcd0cce6323.tar.gz spark-b6072b58bf795093df0e5f8424413fcd0cce6323.tar.bz2 spark-b6072b58bf795093df0e5f8424413fcd0cce6323.zip |
Fixes style, makes "std__-page" consistent, reads only parts of files
Diffstat (limited to 'core/src/main')
4 files changed, 45 insertions, 35 deletions
diff --git a/core/src/main/scala/spark/Utils.scala b/core/src/main/scala/spark/Utils.scala index 64547bbdcd..dfc30469cf 100644 --- a/core/src/main/scala/spark/Utils.scala +++ b/core/src/main/scala/spark/Utils.scala @@ -635,6 +635,20 @@ private object Utils extends Logging { Source.fromBytes(buff).mkString } + /** Return an array containing part of a file from byte 'a' to 'b'. */ + def offsetBytes(path: String, a: Int, b: Int): String = { + val file = new File(path) + val length = file.length() + val buff = new Array[Byte](math.min(b-a, length.toInt)) + val skip = math.max(0, a) + val stream = new FileInputStream(file) + + stream.skip(skip) + stream.read(buff) + stream.close() + Source.fromBytes(buff).mkString + } + /** * Clone an object using a Spark serializer. */ diff --git a/core/src/main/scala/spark/deploy/master/ui/ApplicationPage.scala b/core/src/main/scala/spark/deploy/master/ui/ApplicationPage.scala index ea88421532..cc32728c1c 100644 --- a/core/src/main/scala/spark/deploy/master/ui/ApplicationPage.scala +++ b/core/src/main/scala/spark/deploy/master/ui/ApplicationPage.scala @@ -96,10 +96,10 @@ private[spark] class ApplicationPage(parent: MasterWebUI) { .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stderr</a> </td> <td> - <a href={"%s/logPage?appId=%s&executorId=%s&logType=stdout" - .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stdout</a> - <a href={"%s/logPage?appId=%s&executorId=%s&logType=stderr" - .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stderr</a> + <a href={"%s/logPage?appId=%s&executorId=%s&logType=stdout&offset=0&byteLength=2000" + .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stdout-page</a> + <a href={"%s/logPage?appId=%s&executorId=%s&logType=stderr&offset=0&byteLength=2000" + .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stderr-page</a> </td> </tr> } diff --git a/core/src/main/scala/spark/deploy/worker/ui/IndexPage.scala b/core/src/main/scala/spark/deploy/worker/ui/IndexPage.scala index 7cf98b473e..d532aa9e95 100644 --- a/core/src/main/scala/spark/deploy/worker/ui/IndexPage.scala +++ b/core/src/main/scala/spark/deploy/worker/ui/IndexPage.scala @@ -94,9 +94,9 @@ private[spark] class IndexPage(parent: WorkerWebUI) { .format(executor.appId, executor.execId)}>stderr</a> </td> <td> - <a href={"logPage?appId=%s&executorId=%s&logType=stdout&offset=0&lineLength=20" + <a href={"logPage?appId=%s&executorId=%s&logType=stdout&offset=0&byteLength=2000" .format(executor.appId, executor.execId)}>stdout-page</a> - <a href={"logPage?appId=%s&executorId=%s&logType=stderr&offset=0&lineLength=20" + <a href={"logPage?appId=%s&executorId=%s&logType=stderr&offset=0&byteLength=2000" .format(executor.appId, executor.execId)}>stderr-page</a> </td> </tr> diff --git a/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala b/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala index 5b0c785b00..b018f80c85 100644 --- a/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala +++ b/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala @@ -9,13 +9,13 @@ import javax.servlet.http.HttpServletRequest import org.eclipse.jetty.server.{Handler, Server} +import scala.io.Source._ +import scala.xml._ + import spark.{Utils, Logging} import spark.ui.JettyUtils import spark.ui.JettyUtils._ - -import scala.xml._ import spark.ui.UIUtils -import scala.io.Source._ /** * Web UI server for the standalone worker. @@ -74,42 +74,39 @@ class WorkerWebUI(val worker: ActorRef, val workDir: File, requestedPort: Option val appId = request.getParameter("appId") val executorId = request.getParameter("executorId") val logType = request.getParameter("logType") - val getOffset = request.getParameter("offset") - val getLineLength = request.getParameter("lineLength") + val offset = Option(request.getParameter("offset")).map(_.toInt).getOrElse(0) + + val maxBytes = 1024 * 1024 + val defaultBytes = 100 * 1024 + val byteLength = Option(request.getParameter("byteLength")).flatMap(s => Some(s.toInt)).getOrElse(defaultBytes) + val path = "%s/%s/%s/%s".format(workDir.getPath, appId, executorId, logType) - val source = fromFile(path) - val lines = source.getLines().toArray - val logLength = lines.length - val offset = { - if (getOffset == null) 0 - else if (getOffset.toInt < 0) 0 - else getOffset.toInt - } - val lineLength = { - if (getLineLength == null) 0 - else getLineLength.toInt - } - val logText = "<node>" + lines.slice(offset, offset+lineLength).mkString("\n") + "</node>" - val logXML = XML.loadString(logText) + val logLength = new File(path).length() + val logPageLength = math.min(byteLength, maxBytes) + val logText = <node>{Utils.offsetBytes(path, offset, offset+logPageLength)}</node> + val backButton = if (offset > 0) { - if (offset-lineLength < 0) { - <a href={"?appId=%s&executorId=%s&logType=%s&offset=0&lineLength=%s".format(appId, executorId, logType, lineLength)}> <button style="float:left">back</button> </a> - } - else { - <a href={"?appId=%s&executorId=%s&logType=%s&offset=%s&lineLength=%s".format(appId, executorId, logType, offset-lineLength, lineLength)}> <button style="float:left">back</button> </a> - } + <a href={"?appId=%s&executorId=%s&logType=%s&offset=%s&byteLength=%s" + .format(appId, executorId, logType, math.max(offset-logPageLength, 0), logPageLength)}> + <button style="float:left">back</button> + </a> } else { <button style="float:left" disabled="disabled">back</button> } + val nextButton = - if (offset+lineLength < logLength) { - <a href={"?appId=%s&executorId=%s&logType=%s&offset=%s&lineLength=%s".format(appId, executorId, logType, offset+lineLength, lineLength)}> <button style="float:right">next</button> </a> + if (offset+logPageLength < logLength) { + <a href={"?appId=%s&executorId=%s&logType=%s&offset=%s&byteLength=%s". + format(appId, executorId, logType, offset+logPageLength, logPageLength)}> + <button style="float:right">next</button> + </a> } else { <button style="float:right" disabled="disabled">next</button> } + val content = <html> <body> @@ -117,12 +114,11 @@ class WorkerWebUI(val worker: ActorRef, val workDir: File, requestedPort: Option {backButton} {nextButton} <br></br> - <pre>{logXML}</pre> + <pre>{logText}</pre> {backButton} {nextButton} </body> </html> - source.close() UIUtils.basicSparkPage(content, "Log Page for " + appId) } |