[SPARK-13278][CORE] Launcher fails to start with JDK 9 EA

See http://openjdk.java.net/jeps/223 for more information about the JDK 9 version string scheme.

Author: Claes Redestad <claes.redestad@gmail.com>

Closes #11160 from cl4es/master.

4 files changed, 37 insertions, 7 deletions

diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index bc926c280c..7c6778b065 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -784,8 +784,10 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
         signal(pid, "SIGKILL")
       }
 
-      val v: String = System.getProperty("java.version")
-      if (v >= "1.8.0") {
+      val versionParts = System.getProperty("java.version").split("[+.\\-]+", 3)
+      var majorVersion = versionParts(0).toInt
+      if (majorVersion == 1) majorVersion = versionParts(1).toInt
+      if (majorVersion >= 8) {
         // Java8 added a way to forcibly terminate a process. We'll make sure that works by
         // creating a very misbehaving process. It ignores SIGTERM and has been SIGSTOPed. On
         // older versions of java, this will *not* terminate.
diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
index d30c2ec5f8..e328c8a341 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
@@ -322,11 +322,9 @@ class CommandBuilderUtils {
     if (getJavaVendor() == JavaVendor.IBM) {
       return;
     }
-    String[] version = System.getProperty("java.version").split("\\.");
-    if (Integer.parseInt(version[0]) > 1 || Integer.parseInt(version[1]) > 7) {
+    if (javaMajorVersion(System.getProperty("java.version")) > 7) {
       return;
     }
-
     for (String arg : cmd) {
       if (arg.startsWith("-XX:MaxPermSize=")) {
         return;
@@ -336,4 +334,20 @@ class CommandBuilderUtils {
     cmd.add("-XX:MaxPermSize=256m");
   }
 
+  /**
+   * Get the major version of the java version string supplied. This method
+   * accepts any JEP-223-compliant strings (9-ea, 9+100), as well as legacy
+   * version strings such as 1.7.0_79
+   */
+  static int javaMajorVersion(String javaVersion) {
+    String[] version = javaVersion.split("[+.\\-]+");
+    int major = Integer.parseInt(version[0]);
+    // if major > 1, we're using the JEP-223 version string, e.g., 9-ea, 9+120
+    // otherwise the second number is the major version
+    if (major > 1) {
+      return major;
+    } else {
+      return Integer.parseInt(version[1]);
+    }
+  }
 }
diff --git a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
index bc513ec9b3..4fafc43ef2 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
@@ -87,6 +87,18 @@ public class CommandBuilderUtilsSuite {
     assertEquals("\"a \\\"b\\\" c\"", quoteForCommandString("a \"b\" c"));
   }
 
+  @Test
+  public void testJavaMajorVersion() {
+    assertEquals(6, javaMajorVersion("1.6.0_50"));
+    assertEquals(7, javaMajorVersion("1.7.0_79"));
+    assertEquals(8, javaMajorVersion("1.8.0_66"));
+    assertEquals(9, javaMajorVersion("9-ea"));
+    assertEquals(9, javaMajorVersion("9+100"));
+    assertEquals(9, javaMajorVersion("9"));
+    assertEquals(9, javaMajorVersion("9.1.0"));
+    assertEquals(10, javaMajorVersion("10"));
+  }
+
   private void testOpt(String opts, List<String> expected) {
     assertEquals(String.format("test string failed to parse: [[ %s ]]", opts),
         expected, parseOptionString(opts));
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 6eba58c87c..646efb4d09 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -167,8 +167,10 @@ object SparkBuild extends PomBuild {
     publishLocalBoth <<= Seq(publishLocal in MavenCompile, publishLocal).dependOn,
 
     javacOptions in (Compile, doc) ++= {
-      val Array(major, minor, _) = System.getProperty("java.version").split("\\.", 3)
-      if (major.toInt >= 1 && minor.toInt >= 8) Seq("-Xdoclint:all", "-Xdoclint:-missing") else Seq.empty
+      val versionParts = System.getProperty("java.version").split("[+.\\-]+", 3)
+      var major = versionParts(0).toInt
+      if (major == 1) major = versionParts(1).toInt
+      if (major >= 8) Seq("-Xdoclint:all", "-Xdoclint:-missing") else Seq.empty
     },
 
     javacJVMVersion := "1.7",