From 85a35c68401e171df0b72b172a689d8c4e412199 Mon Sep 17 00:00:00 2001 From: Christoph Grothaus Date: Fri, 15 Feb 2013 14:11:34 +0100 Subject: Fix SPARK-698. From ExecutorRunner, launch java directly instead via the run scripts. --- run | 3 +++ 1 file changed, 3 insertions(+) (limited to 'run') diff --git a/run b/run index 82b1da005a..b5f693f1fa 100755 --- a/run +++ b/run @@ -22,6 +22,8 @@ fi # values for that; it doesn't need a lot if [ "$1" = "spark.deploy.master.Master" -o "$1" = "spark.deploy.worker.Worker" ]; then SPARK_MEM=${SPARK_DAEMON_MEMORY:-512m} + # Backup current SPARK_JAVA_OPTS for use in ExecutorRunner.scala + SPARK_NONDAEMON_JAVA_OPTS=$SPARK_JAVA_OPTS SPARK_JAVA_OPTS=$SPARK_DAEMON_JAVA_OPTS # Empty by default fi @@ -70,6 +72,7 @@ if [ -e $FWDIR/conf/java-opts ] ; then JAVA_OPTS+=" `cat $FWDIR/conf/java-opts`" fi export JAVA_OPTS +# Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in ExecutorRunner.scala! CORE_DIR="$FWDIR/core" REPL_DIR="$FWDIR/repl" -- cgit v1.2.3 From f39f2b7636f52568a556987c8b7f7393299b0351 Mon Sep 17 00:00:00 2001 From: Christoph Grothaus Date: Sun, 24 Feb 2013 21:24:30 +0100 Subject: Incorporate feedback from mateiz: - we do not need getEnvOrEmpty - Instead of saving SPARK_NONDAEMON_JAVA_OPTS, it would be better to modify the scripts to use a different variable name for the JAVA_OPTS they do eventually use --- .../scala/spark/deploy/worker/ExecutorRunner.scala | 24 +++++++--------------- run | 9 ++++---- run2.cmd | 8 ++++---- 3 files changed, 16 insertions(+), 25 deletions(-) (limited to 'run') diff --git a/core/src/main/scala/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/spark/deploy/worker/ExecutorRunner.scala index 214c44fc88..38216ce62f 100644 --- a/core/src/main/scala/spark/deploy/worker/ExecutorRunner.scala +++ b/core/src/main/scala/spark/deploy/worker/ExecutorRunner.scala @@ -75,10 +75,10 @@ private[spark] class ExecutorRunner( def buildCommandSeq(): Seq[String] = { val command = appDesc.command - val runner = if (getEnvOrEmpty("JAVA_HOME") == "") { + val runner = if (System.getenv("JAVA_HOME") == null) { "java" } else { - getEnvOrEmpty("JAVA_HOME") + "/bin/java" + System.getenv("JAVA_HOME") + "/bin/java" } // SPARK-698: do not call the run.cmd script, as process.destroy() // fails to kill a process tree on Windows @@ -91,29 +91,19 @@ private[spark] class ExecutorRunner( * way the JAVA_OPTS are assembled there. */ def buildJavaOpts(): Seq[String] = { - val _javaLibPath = if (getEnvOrEmpty("SPARK_LIBRARY_PATH") == "") { + val _javaLibPath = if (System.getenv("SPARK_LIBRARY_PATH") == null) { "" } else { - "-Djava.library.path=" + getEnvOrEmpty("SPARK_LIBRARY_PATH") + "-Djava.library.path=" + System.getenv("SPARK_LIBRARY_PATH") } Seq("-cp", - getEnvOrEmpty("CLASSPATH"), - // SPARK_JAVA_OPTS is overwritten with SPARK_DAEMON_JAVA_OPTS for running the worker - getEnvOrEmpty("SPARK_NONDAEMON_JAVA_OPTS"), + System.getenv("CLASSPATH"), + System.getenv("SPARK_JAVA_OPTS"), _javaLibPath, "-Xms" + memory.toString + "M", "-Xmx" + memory.toString + "M") - .filter(_ != "") - } - - def getEnvOrEmpty(key: String): String = { - val result = System.getenv(key) - if (result == null) { - "" - } else { - result - } + .filter(_ != null) } /** Spawn a thread that will redirect a given stream to a file */ diff --git a/run b/run index b5f693f1fa..e1482dafbe 100755 --- a/run +++ b/run @@ -22,9 +22,10 @@ fi # values for that; it doesn't need a lot if [ "$1" = "spark.deploy.master.Master" -o "$1" = "spark.deploy.worker.Worker" ]; then SPARK_MEM=${SPARK_DAEMON_MEMORY:-512m} - # Backup current SPARK_JAVA_OPTS for use in ExecutorRunner.scala - SPARK_NONDAEMON_JAVA_OPTS=$SPARK_JAVA_OPTS - SPARK_JAVA_OPTS=$SPARK_DAEMON_JAVA_OPTS # Empty by default + # Do not overwrite SPARK_JAVA_OPTS environment variable in this script + OUR_JAVA_OPTS=$SPARK_DAEMON_JAVA_OPTS # Empty by default +else + OUR_JAVA_OPTS=$SPARK_JAVA_OPTS fi if [ "$SPARK_LAUNCH_WITH_SCALA" == "1" ]; then @@ -64,7 +65,7 @@ fi export SPARK_MEM # Set JAVA_OPTS to be able to load native libraries and to set heap size -JAVA_OPTS="$SPARK_JAVA_OPTS" +JAVA_OPTS="$OUR_JAVA_OPTS" JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH" JAVA_OPTS+=" -Xms$SPARK_MEM -Xmx$SPARK_MEM" # Load extra JAVA_OPTS from conf/java-opts, if it exists diff --git a/run2.cmd b/run2.cmd index a93bbad0b9..8648c0380a 100644 --- a/run2.cmd +++ b/run2.cmd @@ -22,9 +22,9 @@ if "%1"=="spark.deploy.master.Master" set RUNNING_DAEMON=1 if "%1"=="spark.deploy.worker.Worker" set RUNNING_DAEMON=1 if "x%SPARK_DAEMON_MEMORY%" == "x" set SPARK_DAEMON_MEMORY=512m if "%RUNNING_DAEMON%"=="1" set SPARK_MEM=%SPARK_DAEMON_MEMORY% -rem Backup current SPARK_JAVA_OPTS for use in ExecutorRunner.scala -if "%RUNNING_DAEMON%"=="1" set SPARK_NONDAEMON_JAVA_OPTS=%SPARK_JAVA_OPTS% -if "%RUNNING_DAEMON%"=="1" set SPARK_JAVA_OPTS=%SPARK_DAEMON_JAVA_OPTS% +rem Do not overwrite SPARK_JAVA_OPTS environment variable in this script +if "%RUNNING_DAEMON%"=="0" set OUR_JAVA_OPTS=%SPARK_JAVA_OPTS% +if "%RUNNING_DAEMON%"=="1" set OUR_JAVA_OPTS=%SPARK_DAEMON_JAVA_OPTS% rem Check that SCALA_HOME has been specified if not "x%SCALA_HOME%"=="x" goto scala_exists @@ -41,7 +41,7 @@ rem variable so that our process sees it and can report it to Mesos if "x%SPARK_MEM%"=="x" set SPARK_MEM=512m rem Set JAVA_OPTS to be able to load native libraries and to set heap size -set JAVA_OPTS=%SPARK_JAVA_OPTS% -Djava.library.path=%SPARK_LIBRARY_PATH% -Xms%SPARK_MEM% -Xmx%SPARK_MEM% +set JAVA_OPTS=%OUR_JAVA_OPTS% -Djava.library.path=%SPARK_LIBRARY_PATH% -Xms%SPARK_MEM% -Xmx%SPARK_MEM% rem Load extra JAVA_OPTS from conf/java-opts, if it exists if exist "%FWDIR%conf\java-opts.cmd" call "%FWDIR%conf\java-opts.cmd" rem Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in ExecutorRunner.scala! -- cgit v1.2.3