[SPARK-14897][SQL] upgrade to jetty 9.2.16

## What changes were proposed in this pull request?

Since Jetty 8 is EOL (end of life) and has critical security issue [http://www.securityweek.com/critical-vulnerability-found-jetty-web-server], I think upgrading to 9 is necessary. I am using latest 9.2 since 9.3 requires Java 8+.

`javax.servlet` and `derby` were also upgraded since Jetty 9.2 needs corresponding version.

## How was this patch tested?

Manual test and current test cases should cover it.

Author: bomeng <bmeng@us.ibm.com>

Closes #12916 from bomeng/SPARK-14897.

4 files changed, 37 insertions, 31 deletions

diff --git a/core/pom.xml b/core/pom.xml
index 07b5896376..8584b62c0e 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -125,12 +125,15 @@
       <artifactId>jetty-servlet</artifactId>
       <scope>compile</scope>
     </dependency>
-    <!-- Because we mark jetty as provided and shade it, its dependency
-         orbit is ignored, so we explicitly list it here (see SPARK-5557).-->
     <dependency>
-      <groupId>org.eclipse.jetty.orbit</groupId>
-      <artifactId>javax.servlet</artifactId>
-      <version>${orbit.version}</version>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlets</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>javax.servlet</groupId>
+      <artifactId>javax.servlet-api</artifactId>
+      <version>${javaxservlet.version}</version>
     </dependency>
 
     <dependency>
@@ -356,7 +359,7 @@
               <overWriteIfNewer>true</overWriteIfNewer>
               <useSubDirectoryPerType>true</useSubDirectoryPerType>
               <includeArtifactIds>
-                guava,jetty-io,jetty-servlet,jetty-continuation,jetty-http,jetty-plus,jetty-util,jetty-server,jetty-security
+                guava,jetty-io,jetty-servlet,jetty-servlets,jetty-continuation,jetty-http,jetty-plus,jetty-util,jetty-server,jetty-security
               </includeArtifactIds>
               <silent>true</silent>
             </configuration>
diff --git a/core/src/main/scala/org/apache/spark/SSLOptions.scala b/core/src/main/scala/org/apache/spark/SSLOptions.scala
index 719905a2c9..be19179b00 100644
--- a/core/src/main/scala/org/apache/spark/SSLOptions.scala
+++ b/core/src/main/scala/org/apache/spark/SSLOptions.scala
@@ -71,7 +71,7 @@ private[spark] case class SSLOptions(
       keyPassword.foreach(sslContextFactory.setKeyManagerPassword)
       keyStoreType.foreach(sslContextFactory.setKeyStoreType)
       if (needClientAuth) {
-        trustStore.foreach(file => sslContextFactory.setTrustStore(file.getAbsolutePath))
+        trustStore.foreach(file => sslContextFactory.setTrustStorePath(file.getAbsolutePath))
         trustStorePassword.foreach(sslContextFactory.setTrustStorePassword)
         trustStoreType.foreach(sslContextFactory.setTrustStoreType)
       }
diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
index 14244ea571..7e93bfc45e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
@@ -17,13 +17,12 @@
 
 package org.apache.spark.deploy.rest
 
-import java.net.InetSocketAddress
 import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse}
 
 import scala.io.Source
 
 import com.fasterxml.jackson.core.JsonProcessingException
-import org.eclipse.jetty.server.Server
+import org.eclipse.jetty.server.{Server, ServerConnector}
 import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
 import org.eclipse.jetty.util.thread.QueuedThreadPool
 import org.json4s._
@@ -80,10 +79,15 @@ private[spark] abstract class RestSubmissionServer(
    * Return a 2-tuple of the started server and the bound port.
    */
   private def doStart(startPort: Int): (Server, Int) = {
-    val server = new Server(new InetSocketAddress(host, startPort))
     val threadPool = new QueuedThreadPool
     threadPool.setDaemon(true)
-    server.setThreadPool(threadPool)
+    val server = new Server(threadPool)
+
+    val connector = new ServerConnector(server)
+    connector.setHost(host)
+    connector.setPort(startPort)
+    server.addConnector(connector)
+
     val mainHandler = new ServletContextHandler
     mainHandler.setContextPath("/")
     contextToServlet.foreach { case (prefix, servlet) =>
@@ -91,7 +95,7 @@ private[spark] abstract class RestSubmissionServer(
     }
     server.setHandler(mainHandler)
     server.start()
-    val boundPort = server.getConnectors()(0).getLocalPort
+    val boundPort = connector.getLocalPort
     (server, boundPort)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index db24f0319b..6854f7baf7 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -25,11 +25,10 @@ import scala.collection.mutable.ArrayBuffer
 import scala.language.implicitConversions
 import scala.xml.Node
 
-import org.eclipse.jetty.server.{Connector, Request, Server}
+import org.eclipse.jetty.server.{Request, Server, ServerConnector}
 import org.eclipse.jetty.server.handler._
-import org.eclipse.jetty.server.nio.SelectChannelConnector
-import org.eclipse.jetty.server.ssl.SslSelectChannelConnector
 import org.eclipse.jetty.servlet._
+import org.eclipse.jetty.servlets.gzip.GzipHandler
 import org.eclipse.jetty.util.component.LifeCycle
 import org.eclipse.jetty.util.thread.QueuedThreadPool
 import org.json4s.JValue
@@ -243,10 +242,16 @@ private[spark] object JettyUtils extends Logging {
 
     // Bind to the given port, or throw a java.net.BindException if the port is occupied
     def connect(currentPort: Int): (Server, Int) = {
-      val server = new Server
-      val connectors = new ArrayBuffer[Connector]
+      val pool = new QueuedThreadPool
+      if (serverName.nonEmpty) {
+        pool.setName(serverName)
+      }
+      pool.setDaemon(true)
+
+      val server = new Server(pool)
+      val connectors = new ArrayBuffer[ServerConnector]
       // Create a connector on port currentPort to listen for HTTP requests
-      val httpConnector = new SelectChannelConnector()
+      val httpConnector = new ServerConnector(server)
       httpConnector.setPort(currentPort)
       connectors += httpConnector
 
@@ -260,8 +265,9 @@ private[spark] object JettyUtils extends Logging {
           }
         val scheme = "https"
         // Create a connector on port securePort to listen for HTTPS requests
-        val connector = new SslSelectChannelConnector(factory)
+        val connector = new ServerConnector(server, factory)
         connector.setPort(securePort)
+
         connectors += connector
 
         // redirect the HTTP requests to HTTPS port
@@ -269,34 +275,27 @@ private[spark] object JettyUtils extends Logging {
       }
 
       gzipHandlers.foreach(collection.addHandler)
-      connectors.foreach(_.setHost(hostName))
       // As each acceptor and each selector will use one thread, the number of threads should at
       // least be the number of acceptors and selectors plus 1. (See SPARK-13776)
       var minThreads = 1
-      connectors.foreach { c =>
+      connectors.foreach { connector =>
         // Currently we only use "SelectChannelConnector"
-        val connector = c.asInstanceOf[SelectChannelConnector]
         // Limit the max acceptor number to 8 so that we don't waste a lot of threads
-        connector.setAcceptors(math.min(connector.getAcceptors, 8))
+        connector.setAcceptQueueSize(math.min(connector.getAcceptors, 8))
+        connector.setHost(hostName)
         // The number of selectors always equals to the number of acceptors
         minThreads += connector.getAcceptors * 2
       }
       server.setConnectors(connectors.toArray)
-
-      val pool = new QueuedThreadPool
-      if (serverName.nonEmpty) {
-        pool.setName(serverName)
-      }
       pool.setMaxThreads(math.max(pool.getMaxThreads, minThreads))
-      pool.setDaemon(true)
-      server.setThreadPool(pool)
+
       val errorHandler = new ErrorHandler()
       errorHandler.setShowStacks(true)
       server.addBean(errorHandler)
       server.setHandler(collection)
       try {
         server.start()
-        (server, server.getConnectors.head.getLocalPort)
+        (server, httpConnector.getLocalPort)
       } catch {
         case e: Exception =>
           server.stop()