[SPARK-1876] Windows fixes to deal with latest distribution layout changes

- Look for JARs in the right place
- Launch examples the same way as on Unix
- Load datanucleus JARs if they exist
- Don't attempt to parse local paths as URIs in SparkSubmit, since paths with C:\ are not valid URIs
- Also fixed POM exclusion rules for datanucleus (it wasn't properly excluding it, whereas SBT was)

Author: Matei Zaharia <matei@databricks.com>

Closes #819 from mateiz/win-fixes and squashes the following commits:

d558f96 [Matei Zaharia] Fix comment
228577b [Matei Zaharia] Review comments
d3b71c7 [Matei Zaharia] Properly exclude datanucleus files in Maven assembly
144af84 [Matei Zaharia] Update Windows scripts to match latest binary package layout

1 files changed, 23 insertions, 1 deletions

diff --git a/bin/compute-classpath.cmd b/bin/compute-classpath.cmd
index 065553eb31..58710cd1bd 100644
--- a/bin/compute-classpath.cmd
+++ b/bin/compute-classpath.cmd
@@ -20,6 +20,13 @@ rem
 rem This script computes Spark's classpath and prints it to stdout; it's used by both the "run"

 rem script and the ExecutorRunner in standalone cluster mode.

 

+rem If we're called from spark-class2.cmd, it already set enabledelayedexpansion and setting

+rem it here would stop us from affecting its copy of the CLASSPATH variable; otherwise we

+rem need to set it here because we use !datanucleus_jars! below.

+if "%DONT_PRINT_CLASSPATH%"=="1" goto skip_delayed_expansion

+setlocal enabledelayedexpansion

+:skip_delayed_expansion

+

 set SCALA_VERSION=2.10

 

 rem Figure out where the Spark framework is installed

@@ -31,7 +38,7 @@ if exist "%FWDIR%conf\spark-env.cmd" call "%FWDIR%conf\spark-env.cmd"
 rem Build up classpath

 set CLASSPATH=%FWDIR%conf

 if exist "%FWDIR%RELEASE" (

-  for %%d in ("%FWDIR%jars\spark-assembly*.jar") do (

+  for %%d in ("%FWDIR%lib\spark-assembly*.jar") do (

     set ASSEMBLY_JAR=%%d

   )

 ) else (

@@ -42,6 +49,21 @@ if exist "%FWDIR%RELEASE" (
 

 set CLASSPATH=%CLASSPATH%;%ASSEMBLY_JAR%

 

+rem When Hive support is needed, Datanucleus jars must be included on the classpath.

+rem Datanucleus jars do not work if only included in the uber jar as plugin.xml metadata is lost.

+rem Both sbt and maven will populate "lib_managed/jars/" with the datanucleus jars when Spark is

+rem built with Hive, so look for them there.

+if exist "%FWDIR%RELEASE" (

+  set datanucleus_dir=%FWDIR%lib

+) else (

+  set datanucleus_dir=%FWDIR%lib_managed\jars

+)

+set "datanucleus_jars="

+for %%d in ("%datanucleus_dir%\datanucleus-*.jar") do (

+  set datanucleus_jars=!datanucleus_jars!;%%d

+)

+set CLASSPATH=%CLASSPATH%;%datanucleus_jars%

+

 set SPARK_CLASSES=%FWDIR%core\target\scala-%SCALA_VERSION%\classes

 set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%repl\target\scala-%SCALA_VERSION%\classes

 set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%mllib\target\scala-%SCALA_VERSION%\classes