aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorHari Shreedharan <hshreedharan@apache.org>2015-05-20 21:13:10 -0500
committerImran Rashid <irashid@cloudera.com>2015-05-20 21:13:10 -0500
commita70bf06b790add5f279a69607df89ed36155b0e4 (patch)
tree843891f9809c7e2a3a3f7cbdc8bca61e5bc9de93
parent5196efff53af4965ff216a9d5c0f8b2b4fc98652 (diff)
downloadspark-a70bf06b790add5f279a69607df89ed36155b0e4.tar.gz
spark-a70bf06b790add5f279a69607df89ed36155b0e4.tar.bz2
spark-a70bf06b790add5f279a69607df89ed36155b0e4.zip
[SPARK-7750] [WEBUI] Rename endpoints from `json` to `api` to allow fu…
…rther extension to non-json outputs too. Author: Hari Shreedharan <hshreedharan@apache.org> Closes #6273 from harishreedharan/json-to-api and squashes the following commits: e14b73b [Hari Shreedharan] Rename `getJsonServlet` to `getServletHandler` i 42f8acb [Hari Shreedharan] Import order fixes. 2ef852f [Hari Shreedharan] [SPARK-7750][WebUI] Rename endpoints from `json` to `api` to allow further extension to non-json outputs too.
-rw-r--r--core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala5
-rw-r--r--core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala5
-rw-r--r--core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala (renamed from core/src/main/scala/org/apache/spark/status/api/v1/JsonRootResource.scala)8
-rw-r--r--core/src/main/scala/org/apache/spark/ui/SparkUI.scala5
-rw-r--r--core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala4
-rw-r--r--core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala16
-rw-r--r--docs/monitoring.md10
7 files changed, 28 insertions, 25 deletions
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index 517cbe5176..5a0eb585a9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -25,7 +25,8 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
import org.apache.spark.{Logging, SecurityManager, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.status.api.v1.{ApplicationInfo, ApplicationsListResource, JsonRootResource, UIRoot}
+import org.apache.spark.status.api.v1.{ApiRootResource, ApplicationInfo, ApplicationsListResource,
+ UIRoot}
import org.apache.spark.ui.{SparkUI, UIUtils, WebUI}
import org.apache.spark.ui.JettyUtils._
import org.apache.spark.util.{SignalLogger, Utils}
@@ -125,7 +126,7 @@ class HistoryServer(
def initialize() {
attachPage(new HistoryPage(this))
- attachHandler(JsonRootResource.getJsonServlet(this))
+ attachHandler(ApiRootResource.getServletHandler(this))
attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static"))
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
index eb26e9f99c..2111a8581f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
@@ -19,7 +19,8 @@ package org.apache.spark.deploy.master.ui
import org.apache.spark.Logging
import org.apache.spark.deploy.master.Master
-import org.apache.spark.status.api.v1.{ApplicationsListResource, ApplicationInfo, JsonRootResource, UIRoot}
+import org.apache.spark.status.api.v1.{ApiRootResource, ApplicationsListResource, ApplicationInfo,
+ UIRoot}
import org.apache.spark.ui.{SparkUI, WebUI}
import org.apache.spark.ui.JettyUtils._
import org.apache.spark.util.RpcUtils
@@ -47,7 +48,7 @@ class MasterWebUI(val master: Master, requestedPort: Int)
attachPage(new HistoryNotFoundPage(this))
attachPage(masterPage)
attachHandler(createStaticHandler(MasterWebUI.STATIC_RESOURCE_DIR, "/static"))
- attachHandler(JsonRootResource.getJsonServlet(this))
+ attachHandler(ApiRootResource.getServletHandler(this))
attachHandler(createRedirectHandler(
"/app/kill", "/", masterPage.handleAppKillRequest, httpMethods = Set("POST")))
attachHandler(createRedirectHandler(
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/JsonRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
index c3ec45f546..bf2cc2e72f 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/JsonRootResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
@@ -39,7 +39,7 @@ import org.apache.spark.ui.SparkUI
* HistoryServerSuite.
*/
@Path("/v1")
-private[v1] class JsonRootResource extends UIRootFromServletContext {
+private[v1] class ApiRootResource extends UIRootFromServletContext {
@Path("applications")
def getApplicationList(): ApplicationListResource = {
@@ -166,11 +166,11 @@ private[v1] class JsonRootResource extends UIRootFromServletContext {
}
-private[spark] object JsonRootResource {
+private[spark] object ApiRootResource {
- def getJsonServlet(uiRoot: UIRoot): ServletContextHandler = {
+ def getServletHandler(uiRoot: UIRoot): ServletContextHandler = {
val jerseyContext = new ServletContextHandler(ServletContextHandler.NO_SESSIONS)
- jerseyContext.setContextPath("/json")
+ jerseyContext.setContextPath("/api")
val holder:ServletHolder = new ServletHolder(classOf[ServletContainer])
holder.setInitParameter("com.sun.jersey.config.property.resourceConfigClass",
"com.sun.jersey.api.core.PackagesResourceConfig")
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index bfe4a180e8..0b11e914bb 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -19,7 +19,8 @@ package org.apache.spark.ui
import java.util.Date
-import org.apache.spark.status.api.v1.{ApplicationAttemptInfo, ApplicationInfo, JsonRootResource, UIRoot}
+import org.apache.spark.status.api.v1.{ApiRootResource, ApplicationAttemptInfo, ApplicationInfo,
+ UIRoot}
import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkContext}
import org.apache.spark.scheduler._
import org.apache.spark.storage.StorageStatusListener
@@ -64,7 +65,7 @@ private[spark] class SparkUI private (
attachTab(new ExecutorsTab(this))
attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static"))
attachHandler(createRedirectHandler("/", "/jobs", basePath = basePath))
- attachHandler(JsonRootResource.getJsonServlet(this))
+ attachHandler(ApiRootResource.getServletHandler(this))
// This should be POST only, but, the YARN AM proxy won't proxy POSTs
attachHandler(createRedirectHandler(
"/stages/stage/kill", "/stages", stagesTab.handleKillRequest,
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 318ab5dbc4..4adb5122bc 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -198,11 +198,11 @@ class HistoryServerSuite extends FunSuite with BeforeAndAfter with Matchers with
}
def getContentAndCode(path: String, port: Int = port): (Int, Option[String], Option[String]) = {
- HistoryServerSuite.getContentAndCode(new URL(s"http://localhost:$port/json/v1/$path"))
+ HistoryServerSuite.getContentAndCode(new URL(s"http://localhost:$port/api/v1/$path"))
}
def getUrl(path: String): String = {
- HistoryServerSuite.getUrl(new URL(s"http://localhost:$port/json/v1/$path"))
+ HistoryServerSuite.getUrl(new URL(s"http://localhost:$port/api/v1/$path"))
}
def generateExpectation(name: String, path: String): Unit = {
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 117b2c3960..b6f5accef0 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -497,7 +497,7 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers with Before
goToUi(sc, "/jobs/job/?id=7")
find("no-info").get.text should be ("No information to display for job 7")
- val badJob = HistoryServerSuite.getContentAndCode(jsonUrl(sc.ui.get, "jobs/7"))
+ val badJob = HistoryServerSuite.getContentAndCode(apiUrl(sc.ui.get, "jobs/7"))
badJob._1 should be (HttpServletResponse.SC_NOT_FOUND)
badJob._2 should be (None)
badJob._3 should be (Some("unknown job: 7"))
@@ -540,18 +540,18 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers with Before
goToUi(sc, "/stages/stage/?id=12&attempt=0")
find("no-info").get.text should be ("No information to display for Stage 12 (Attempt 0)")
- val badStage = HistoryServerSuite.getContentAndCode(jsonUrl(sc.ui.get,"stages/12/0"))
+ val badStage = HistoryServerSuite.getContentAndCode(apiUrl(sc.ui.get,"stages/12/0"))
badStage._1 should be (HttpServletResponse.SC_NOT_FOUND)
badStage._2 should be (None)
badStage._3 should be (Some("unknown stage: 12"))
- val badAttempt = HistoryServerSuite.getContentAndCode(jsonUrl(sc.ui.get,"stages/19/15"))
+ val badAttempt = HistoryServerSuite.getContentAndCode(apiUrl(sc.ui.get,"stages/19/15"))
badAttempt._1 should be (HttpServletResponse.SC_NOT_FOUND)
badAttempt._2 should be (None)
badAttempt._3 should be (Some("unknown attempt for stage 19. Found attempts: [0]"))
val badStageAttemptList = HistoryServerSuite.getContentAndCode(
- jsonUrl(sc.ui.get, "stages/12"))
+ apiUrl(sc.ui.get, "stages/12"))
badStageAttemptList._1 should be (HttpServletResponse.SC_NOT_FOUND)
badStageAttemptList._2 should be (None)
badStageAttemptList._3 should be (Some("unknown stage: 12"))
@@ -561,7 +561,7 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers with Before
test("live UI json application list") {
withSpark(newSparkContext()) { sc =>
val appListRawJson = HistoryServerSuite.getUrl(new URL(
- sc.ui.get.appUIAddress + "/json/v1/applications"))
+ sc.ui.get.appUIAddress + "/api/v1/applications"))
val appListJsonAst = JsonMethods.parse(appListRawJson)
appListJsonAst.children.length should be (1)
val attempts = (appListJsonAst \ "attempts").children
@@ -587,10 +587,10 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers with Before
}
def getJson(ui: SparkUI, path: String): JValue = {
- JsonMethods.parse(HistoryServerSuite.getUrl(jsonUrl(ui, path)))
+ JsonMethods.parse(HistoryServerSuite.getUrl(apiUrl(ui, path)))
}
- def jsonUrl(ui: SparkUI, path: String): URL = {
- new URL(ui.appUIAddress + "/json/v1/applications/test/" + path)
+ def apiUrl(ui: SparkUI, path: String): URL = {
+ new URL(ui.appUIAddress + "/api/v1/applications/test/" + path)
}
}
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 1e0fc15086..e750184990 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -178,9 +178,9 @@ Note that the history server only displays completed Spark jobs. One way to sign
In addition to viewing the metrics in the UI, they are also available as JSON. This gives developers
an easy way to create new visualizations and monitoring tools for Spark. The JSON is available for
-both running applications, and in the history server. The endpoints are mounted at `/json/v1`. Eg.,
-for the history server, they would typically be accessible at `http://<server-url>:18080/json/v1`, and
-for a running application, at `http://localhost:4040/json/v1`.
+both running applications, and in the history server. The endpoints are mounted at `/api/v1`. Eg.,
+for the history server, they would typically be accessible at `http://<server-url>:18080/api/v1`, and
+for a running application, at `http://localhost:4040/api/v1`.
<table class="table">
<tr><th>Endpoint</th><th>Meaning</th></tr>
@@ -240,12 +240,12 @@ These endpoints have been strongly versioned to make it easier to develop applic
* Individual fields will never be removed for any given endpoint
* New endpoints may be added
* New fields may be added to existing endpoints
-* New versions of the api may be added in the future at a separate endpoint (eg., `json/v2`). New versions are *not* required to be backwards compatible.
+* New versions of the api may be added in the future at a separate endpoint (eg., `api/v2`). New versions are *not* required to be backwards compatible.
* Api versions may be dropped, but only after at least one minor release of co-existing with a new api version
Note that even when examining the UI of a running applications, the `applications/[app-id]` portion is
still required, though there is only one application available. Eg. to see the list of jobs for the
-running app, you would go to `http://localhost:4040/json/v1/applications/[app-id]/jobs`. This is to
+running app, you would go to `http://localhost:4040/api/v1/applications/[app-id]/jobs`. This is to
keep the paths consistent in both modes.
# Metrics