Merge branch 'master' into mos-bt

269 files changed, 199 insertions, 117 deletions

diff --git a/.gitignore b/.gitignore
index 5abdec5d50..e6cdca77e6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,10 +1,26 @@
 *~
 *.swp
-build
-work
+*.iml
+.idea/
+/build/
+work/
+out/
 .DS_Store
 third_party/libmesos.so
 third_party/libmesos.dylib
 conf/java-opts
 conf/spark-env.sh
 conf/log4j.properties
+target/
+reports/
+.project
+.classpath
+.scala_dependencies
+lib_managed/
+src_managed/
+project/boot/
+project/plugins/project/build.properties
+project/build/target/
+project/plugins/target/
+project/plugins/lib_managed/
+project/plugins/src_managed/
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 762d752fda..0000000000
--- a/Makefile
+++ /dev/null
@@ -1,73 +0,0 @@
-EMPTY =
-SPACE = $(EMPTY) $(EMPTY)
-
-# Build up classpath by concatenating some strings
-JARS = third_party/mesos.jar
-JARS += third_party/asm-3.2/lib/all/asm-all-3.2.jar
-JARS += third_party/colt.jar
-JARS += third_party/guava-r07/guava-r07.jar
-JARS += third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar
-JARS += third_party/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
-JARS += third_party/scalatest-1.2/scalatest-1.2.jar
-JARS += third_party/scalacheck_2.8.0-1.7.jar
-JARS += third_party/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
-JARS += third_party/jetty-7.1.6.v20100715/servlet-api-2.5.jar
-JARS += third_party/apache-log4j-1.2.16/log4j-1.2.16.jar
-JARS += third_party/slf4j-1.6.1/slf4j-api-1.6.1.jar
-JARS += third_party/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
-JARS += third_party/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
-
-CLASSPATH = $(subst $(SPACE),:,$(JARS))
-
-SCALA_SOURCES =  src/examples/*.scala src/scala/spark/*.scala src/scala/spark/repl/*.scala
-SCALA_SOURCES += src/test/spark/*.scala src/test/spark/repl/*.scala
-
-
-ifeq ($(USE_FSC),1)
-  COMPILER_NAME = fsc
-else
-  COMPILER_NAME = scalac
-endif
-
-ifeq ($(SCALA_HOME),)
-  COMPILER = $(COMPILER_NAME)
-else
-  COMPILER = $(SCALA_HOME)/bin/$(COMPILER_NAME)
-endif
-
-CONF_FILES = conf/spark-env.sh conf/log4j.properties conf/java-opts
-
-all: scala conf-files
-
-build/classes:
-	mkdir -p build/classes
-
-scala: build/classes
-	$(COMPILER) -d build/classes -classpath build/classes:$(CLASSPATH) $(SCALA_SOURCES)
-
-jar: build/spark.jar build/spark-dep.jar
-
-dep-jar: build/spark-dep.jar
-
-build/spark.jar: scala
-	jar cf build/spark.jar -C build/classes spark
-
-build/spark-dep.jar:
-	mkdir -p build/dep
-	cd build/dep &&	for i in $(JARS); do jar xf ../../$$i; done
-	jar cf build/spark-dep.jar -C build/dep .
-
-conf-files: $(CONF_FILES)
-
-$(CONF_FILES): %: | %.template
-	cp $@.template $@
-
-test: all
-	./alltests
-
-default: all
-
-clean:
-	rm -rf build
-
-.phony: default all clean scala jar dep-jar conf-files
diff --git a/README b/README
index f084f22a1f..3fbf8c98de 100644
--- a/README
+++ b/README
@@ -1,24 +1,32 @@
+ONLINE DOCUMENTATION
+
+You can find the latest Spark documentation, including a programming guide,
+on the project wiki at http://github.com/mesos/spark/wiki. This file only
+contains basic setup instructions.
+
+
+
 BUILDING
 
-Spark requires Scala 2.8. This version has been tested with 2.8.0.final.
+Spark requires Scala 2.8. This version has been tested with 2.8.1.final.
 
-To build and run Spark, you will need to have Scala's bin in your $PATH,
-or you will need to set the SCALA_HOME environment variable to point
-to where you've installed Scala. Scala must be accessible through one
-of these methods on Mesos slave nodes as well as on the master.
+The project is built using Simple Build Tool (SBT), which is packaged with it.
+To build Spark and its example programs, run sbt/sbt compile.
 
-To build Spark and the example programs, run make.
+To run Spark, you will need to have Scala's bin in your $PATH, or you
+will need to set the SCALA_HOME environment variable to point to where
+you've installed Scala. Scala must be accessible through one of these
+methods on Mesos slave nodes as well as on the master.
 
 To run one of the examples, use ./run <class> <params>. For example,
-./run SparkLR will run the Logistic Regression example. Each of the
-example programs prints usage help if no params are given.
+./run spark.examples.SparkLR will run the Logistic Regression example.
+Each of the example programs prints usage help if no params are given.
 
 All of the Spark samples take a <host> parameter that is the Mesos master
 to connect to. This can be a Mesos URL, or "local" to run locally with one
 thread, or "local[N]" to run locally with N threads.
 
-Tip: If you are building Spark and examples repeatedly, export USE_FSC=1
-to have the Makefile use the fsc compiler daemon instead of scalac.
+
 
 CONFIGURATION
 
diff --git a/alltests b/alltests
deleted file mode 100755
index cd11604855..0000000000
--- a/alltests
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-FWDIR="`dirname $0`"
-if [ "x$SPARK_MEM" == "x" ]; then
-  export SPARK_MEM=500m
-fi
-RESULTS_DIR="$FWDIR/build/test_results"
-if [ -d $RESULTS_DIR ]; then
-  rm -r $RESULTS_DIR
-fi
-mkdir -p $RESULTS_DIR
-$FWDIR/run org.scalatest.tools.Runner -p $FWDIR/build/classes -u $RESULTS_DIR -o $@
diff --git a/third_party/apache-log4j-1.2.16/log4j-1.2.16.jar b/core/lib/apache-log4j-1.2.16/log4j-1.2.16.jar
index 3f9d847618..3f9d847618 100644
--- a/third_party/apache-log4j-1.2.16/log4j-1.2.16.jar
+++ b/core/lib/apache-log4j-1.2.16/log4j-1.2.16.jar
diff --git a/third_party/asm-3.2/.DS_Store b/core/lib/asm-3.2/.DS_Store
index 52b0f12a32..52b0f12a32 100644
--- a/third_party/asm-3.2/.DS_Store
+++ b/core/lib/asm-3.2/.DS_Store
diff --git a/third_party/asm-3.2/lib/all/README.txt b/core/lib/asm-3.2/lib/all/README.txt
index d7c96a5edb..d7c96a5edb 100644
--- a/third_party/asm-3.2/lib/all/README.txt
+++ b/core/lib/asm-3.2/lib/all/README.txt
diff --git a/third_party/asm-3.2/lib/all/asm-all-3.2.jar b/core/lib/asm-3.2/lib/all/asm-all-3.2.jar
index d0ad60ed0a..d0ad60ed0a 100644
--- a/third_party/asm-3.2/lib/all/asm-all-3.2.jar
+++ b/core/lib/asm-3.2/lib/all/asm-all-3.2.jar
diff --git a/third_party/asm-3.2/lib/all/asm-all-3.2.pom b/core/lib/asm-3.2/lib/all/asm-all-3.2.pom
index 9899a54c3b..9899a54c3b 100644
--- a/third_party/asm-3.2/lib/all/asm-all-3.2.pom
+++ b/core/lib/asm-3.2/lib/all/asm-all-3.2.pom
diff --git a/third_party/asm-3.2/lib/all/asm-debug-all-3.2.jar b/core/lib/asm-3.2/lib/all/asm-debug-all-3.2.jar
index 94b8549142..94b8549142 100644
--- a/third_party/asm-3.2/lib/all/asm-debug-all-3.2.jar
+++ b/core/lib/asm-3.2/lib/all/asm-debug-all-3.2.jar
diff --git a/third_party/asm-3.2/lib/all/asm-debug-all-3.2.pom b/core/lib/asm-3.2/lib/all/asm-debug-all-3.2.pom
index 9899a54c3b..9899a54c3b 100644
--- a/third_party/asm-3.2/lib/all/asm-debug-all-3.2.pom
+++ b/core/lib/asm-3.2/lib/all/asm-debug-all-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-3.2.jar b/core/lib/asm-3.2/lib/asm-3.2.jar
index 334e7fdc7f..334e7fdc7f 100644
--- a/third_party/asm-3.2/lib/asm-3.2.jar
+++ b/core/lib/asm-3.2/lib/asm-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-3.2.pom b/core/lib/asm-3.2/lib/asm-3.2.pom
index c714db09b2..c714db09b2 100644
--- a/third_party/asm-3.2/lib/asm-3.2.pom
+++ b/core/lib/asm-3.2/lib/asm-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-analysis-3.2.jar b/core/lib/asm-3.2/lib/asm-analysis-3.2.jar
index 40ee3151cb..40ee3151cb 100644
--- a/third_party/asm-3.2/lib/asm-analysis-3.2.jar
+++ b/core/lib/asm-3.2/lib/asm-analysis-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-analysis-3.2.pom b/core/lib/asm-3.2/lib/asm-analysis-3.2.pom
index b3933387af..b3933387af 100644
--- a/third_party/asm-3.2/lib/asm-analysis-3.2.pom
+++ b/core/lib/asm-3.2/lib/asm-analysis-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-commons-3.2.jar b/core/lib/asm-3.2/lib/asm-commons-3.2.jar
index 8dfed0a9b7..8dfed0a9b7 100644
--- a/third_party/asm-3.2/lib/asm-commons-3.2.jar
+++ b/core/lib/asm-3.2/lib/asm-commons-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-commons-3.2.pom b/core/lib/asm-3.2/lib/asm-commons-3.2.pom
index 8517715b4a..8517715b4a 100644
--- a/third_party/asm-3.2/lib/asm-commons-3.2.pom
+++ b/core/lib/asm-3.2/lib/asm-commons-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-parent-3.2.pom b/core/lib/asm-3.2/lib/asm-parent-3.2.pom
index c220347f6a..c220347f6a 100644
--- a/third_party/asm-3.2/lib/asm-parent-3.2.pom
+++ b/core/lib/asm-3.2/lib/asm-parent-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-tree-3.2.jar b/core/lib/asm-3.2/lib/asm-tree-3.2.jar
index b21fb86a92..b21fb86a92 100644
--- a/third_party/asm-3.2/lib/asm-tree-3.2.jar
+++ b/core/lib/asm-3.2/lib/asm-tree-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-tree-3.2.pom b/core/lib/asm-3.2/lib/asm-tree-3.2.pom
index 9f454528f4..9f454528f4 100644
--- a/third_party/asm-3.2/lib/asm-tree-3.2.pom
+++ b/core/lib/asm-3.2/lib/asm-tree-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-util-3.2.jar b/core/lib/asm-3.2/lib/asm-util-3.2.jar
index 499d229034..499d229034 100644
--- a/third_party/asm-3.2/lib/asm-util-3.2.jar
+++ b/core/lib/asm-3.2/lib/asm-util-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-util-3.2.pom b/core/lib/asm-3.2/lib/asm-util-3.2.pom
index e302b0f356..e302b0f356 100644
--- a/third_party/asm-3.2/lib/asm-util-3.2.pom
+++ b/core/lib/asm-3.2/lib/asm-util-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-xml-3.2.jar b/core/lib/asm-3.2/lib/asm-xml-3.2.jar
index 31b31b56fe..31b31b56fe 100644
--- a/third_party/asm-3.2/lib/asm-xml-3.2.jar
+++ b/core/lib/asm-3.2/lib/asm-xml-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-xml-3.2.pom b/core/lib/asm-3.2/lib/asm-xml-3.2.pom
index 0f3de1f2ab..0f3de1f2ab 100644
--- a/third_party/asm-3.2/lib/asm-xml-3.2.pom
+++ b/core/lib/asm-3.2/lib/asm-xml-3.2.pom
diff --git a/third_party/colt.jar b/core/lib/colt.jar
index a7192f68b3..a7192f68b3 100644
--- a/third_party/colt.jar
+++ b/core/lib/colt.jar
diff --git a/third_party/compress-lzf-0.6.0/LICENSE b/core/lib/compress-lzf-0.6.0/LICENSE
index c5da4e1348..c5da4e1348 100644
--- a/third_party/compress-lzf-0.6.0/LICENSE
+++ b/core/lib/compress-lzf-0.6.0/LICENSE
diff --git a/third_party/compress-lzf-0.6.0/compress-lzf-0.6.0.jar b/core/lib/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
index 6cb5c4c92b..6cb5c4c92b 100644
--- a/third_party/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
+++ b/core/lib/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
diff --git a/third_party/guava-r07/COPYING b/core/lib/guava-r07/COPYING
index d645695673..d645695673 100644
--- a/third_party/guava-r07/COPYING
+++ b/core/lib/guava-r07/COPYING
diff --git a/third_party/guava-r07/README b/core/lib/guava-r07/README
index a0e832dd54..a0e832dd54 100644
--- a/third_party/guava-r07/README
+++ b/core/lib/guava-r07/README
diff --git a/third_party/guava-r07/guava-r07.jar b/core/lib/guava-r07/guava-r07.jar
index a6c9ce02df..a6c9ce02df 100644
--- a/third_party/guava-r07/guava-r07.jar
+++ b/core/lib/guava-r07/guava-r07.jar
diff --git a/third_party/hadoop-0.20.0/.DS_Store b/core/lib/hadoop-0.20.0/.DS_Store
index 81f4e05e09..81f4e05e09 100644
--- a/third_party/hadoop-0.20.0/.DS_Store
+++ b/core/lib/hadoop-0.20.0/.DS_Store
diff --git a/third_party/hadoop-0.20.0/CHANGES.txt b/core/lib/hadoop-0.20.0/CHANGES.txt
index 95c8b5c08b..95c8b5c08b 100644
--- a/third_party/hadoop-0.20.0/CHANGES.txt
+++ b/core/lib/hadoop-0.20.0/CHANGES.txt
diff --git a/third_party/hadoop-0.20.0/LICENSE.txt b/core/lib/hadoop-0.20.0/LICENSE.txt
index 59bcdbc978..59bcdbc978 100644
--- a/third_party/hadoop-0.20.0/LICENSE.txt
+++ b/core/lib/hadoop-0.20.0/LICENSE.txt
diff --git a/third_party/hadoop-0.20.0/NOTICE.txt b/core/lib/hadoop-0.20.0/NOTICE.txt
index 62fc5816c9..62fc5816c9 100644
--- a/third_party/hadoop-0.20.0/NOTICE.txt
+++ b/core/lib/hadoop-0.20.0/NOTICE.txt
diff --git a/third_party/hadoop-0.20.0/README.txt b/core/lib/hadoop-0.20.0/README.txt
index 148cd31c86..148cd31c86 100644
--- a/third_party/hadoop-0.20.0/README.txt
+++ b/core/lib/hadoop-0.20.0/README.txt
diff --git a/third_party/hadoop-0.20.0/bin/hadoop b/core/lib/hadoop-0.20.0/bin/hadoop
index 273549f138..273549f138 100755
--- a/third_party/hadoop-0.20.0/bin/hadoop
+++ b/core/lib/hadoop-0.20.0/bin/hadoop
diff --git a/third_party/hadoop-0.20.0/bin/hadoop-config.sh b/core/lib/hadoop-0.20.0/bin/hadoop-config.sh
index 1f9d52da79..1f9d52da79 100755
--- a/third_party/hadoop-0.20.0/bin/hadoop-config.sh
+++ b/core/lib/hadoop-0.20.0/bin/hadoop-config.sh
diff --git a/third_party/hadoop-0.20.0/bin/hadoop-daemon.sh b/core/lib/hadoop-0.20.0/bin/hadoop-daemon.sh
index e10390a9e2..e10390a9e2 100755
--- a/third_party/hadoop-0.20.0/bin/hadoop-daemon.sh
+++ b/core/lib/hadoop-0.20.0/bin/hadoop-daemon.sh
diff --git a/third_party/hadoop-0.20.0/bin/hadoop-daemons.sh b/core/lib/hadoop-0.20.0/bin/hadoop-daemons.sh
index 894d8ab11a..894d8ab11a 100755
--- a/third_party/hadoop-0.20.0/bin/hadoop-daemons.sh
+++ b/core/lib/hadoop-0.20.0/bin/hadoop-daemons.sh
diff --git a/third_party/hadoop-0.20.0/bin/rcc b/core/lib/hadoop-0.20.0/bin/rcc
index a39745be6f..a39745be6f 100755
--- a/third_party/hadoop-0.20.0/bin/rcc
+++ b/core/lib/hadoop-0.20.0/bin/rcc
diff --git a/third_party/hadoop-0.20.0/bin/slaves.sh b/core/lib/hadoop-0.20.0/bin/slaves.sh
index fc9f720be7..fc9f720be7 100755
--- a/third_party/hadoop-0.20.0/bin/slaves.sh
+++ b/core/lib/hadoop-0.20.0/bin/slaves.sh
diff --git a/third_party/hadoop-0.20.0/bin/start-all.sh b/core/lib/hadoop-0.20.0/bin/start-all.sh
index b1eefc8fbe..b1eefc8fbe 100755
--- a/third_party/hadoop-0.20.0/bin/start-all.sh
+++ b/core/lib/hadoop-0.20.0/bin/start-all.sh
diff --git a/third_party/hadoop-0.20.0/bin/start-balancer.sh b/core/lib/hadoop-0.20.0/bin/start-balancer.sh
index e8c93f90ca..e8c93f90ca 100755
--- a/third_party/hadoop-0.20.0/bin/start-balancer.sh
+++ b/core/lib/hadoop-0.20.0/bin/start-balancer.sh
diff --git a/third_party/hadoop-0.20.0/bin/start-dfs.sh b/core/lib/hadoop-0.20.0/bin/start-dfs.sh
index bda2035a2b..bda2035a2b 100755
--- a/third_party/hadoop-0.20.0/bin/start-dfs.sh
+++ b/core/lib/hadoop-0.20.0/bin/start-dfs.sh
diff --git a/third_party/hadoop-0.20.0/bin/start-mapred.sh b/core/lib/hadoop-0.20.0/bin/start-mapred.sh
index b64c8f51d8..b64c8f51d8 100755
--- a/third_party/hadoop-0.20.0/bin/start-mapred.sh
+++ b/core/lib/hadoop-0.20.0/bin/start-mapred.sh
diff --git a/third_party/hadoop-0.20.0/bin/stop-all.sh b/core/lib/hadoop-0.20.0/bin/stop-all.sh
index 033f2fe8d8..033f2fe8d8 100755
--- a/third_party/hadoop-0.20.0/bin/stop-all.sh
+++ b/core/lib/hadoop-0.20.0/bin/stop-all.sh
diff --git a/third_party/hadoop-0.20.0/bin/stop-balancer.sh b/core/lib/hadoop-0.20.0/bin/stop-balancer.sh
index 483a9c2549..483a9c2549 100755
--- a/third_party/hadoop-0.20.0/bin/stop-balancer.sh
+++ b/core/lib/hadoop-0.20.0/bin/stop-balancer.sh
diff --git a/third_party/hadoop-0.20.0/bin/stop-dfs.sh b/core/lib/hadoop-0.20.0/bin/stop-dfs.sh
index 14fe61d17e..14fe61d17e 100755
--- a/third_party/hadoop-0.20.0/bin/stop-dfs.sh
+++ b/core/lib/hadoop-0.20.0/bin/stop-dfs.sh
diff --git a/third_party/hadoop-0.20.0/bin/stop-mapred.sh b/core/lib/hadoop-0.20.0/bin/stop-mapred.sh
index aa51c1f87b..aa51c1f87b 100755
--- a/third_party/hadoop-0.20.0/bin/stop-mapred.sh
+++ b/core/lib/hadoop-0.20.0/bin/stop-mapred.sh
diff --git a/third_party/hadoop-0.20.0/build.xml b/core/lib/hadoop-0.20.0/build.xml
index 68932d7d9f..68932d7d9f 100644
--- a/third_party/hadoop-0.20.0/build.xml
+++ b/core/lib/hadoop-0.20.0/build.xml
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/Pipes.hh b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/Pipes.hh
index 9a785d966a..9a785d966a 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/Pipes.hh
+++ b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/Pipes.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/SerialUtils.hh b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/SerialUtils.hh
index 16cbab65b2..16cbab65b2 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/SerialUtils.hh
+++ b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/SerialUtils.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/StringUtils.hh b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/StringUtils.hh
index 4720172725..4720172725 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/StringUtils.hh
+++ b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/StringUtils.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/TemplateFactory.hh b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/TemplateFactory.hh
index 22e10ae56f..22e10ae56f 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/TemplateFactory.hh
+++ b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/TemplateFactory.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooppipes.a b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooppipes.a
index be303140cb..be303140cb 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooppipes.a
+++ b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooppipes.a
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooputils.a b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooputils.a
index 8a0aded98e..8a0aded98e 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooputils.a
+++ b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooputils.a
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/Pipes.hh b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/Pipes.hh
index 9a785d966a..9a785d966a 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/Pipes.hh
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/Pipes.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/SerialUtils.hh b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/SerialUtils.hh
index 16cbab65b2..16cbab65b2 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/SerialUtils.hh
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/SerialUtils.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/StringUtils.hh b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/StringUtils.hh
index 4720172725..4720172725 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/StringUtils.hh
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/StringUtils.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/TemplateFactory.hh b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/TemplateFactory.hh
index 22e10ae56f..22e10ae56f 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/TemplateFactory.hh
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/TemplateFactory.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooppipes.a b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooppipes.a
index 73debcec92..73debcec92 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooppipes.a
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooppipes.a
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooputils.a b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooputils.a
index 6753169b8a..6753169b8a 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooputils.a
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooputils.a
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.la b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.la
index b6ce94229f..b6ce94229f 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.la
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.la
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so
index 358d582d43..358d582d43 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0 b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0
index 358d582d43..358d582d43 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0.0.0 b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0.0.0
index 358d582d43..358d582d43 100644
--- a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0.0.0
+++ b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0.0.0
diff --git a/third_party/hadoop-0.20.0/conf/capacity-scheduler.xml b/core/lib/hadoop-0.20.0/conf/capacity-scheduler.xml
index d22a3964b4..d22a3964b4 100644
--- a/third_party/hadoop-0.20.0/conf/capacity-scheduler.xml
+++ b/core/lib/hadoop-0.20.0/conf/capacity-scheduler.xml
diff --git a/third_party/hadoop-0.20.0/conf/configuration.xsl b/core/lib/hadoop-0.20.0/conf/configuration.xsl
index 377cdbeb93..377cdbeb93 100644
--- a/third_party/hadoop-0.20.0/conf/configuration.xsl
+++ b/core/lib/hadoop-0.20.0/conf/configuration.xsl
diff --git a/third_party/hadoop-0.20.0/conf/core-site.xml b/core/lib/hadoop-0.20.0/conf/core-site.xml
index 970c8fe0e8..970c8fe0e8 100644
--- a/third_party/hadoop-0.20.0/conf/core-site.xml
+++ b/core/lib/hadoop-0.20.0/conf/core-site.xml
diff --git a/third_party/hadoop-0.20.0/conf/hadoop-env.sh b/core/lib/hadoop-0.20.0/conf/hadoop-env.sh
index ada5bef1c7..ada5bef1c7 100644
--- a/third_party/hadoop-0.20.0/conf/hadoop-env.sh
+++ b/core/lib/hadoop-0.20.0/conf/hadoop-env.sh
diff --git a/third_party/hadoop-0.20.0/conf/hadoop-metrics.properties b/core/lib/hadoop-0.20.0/conf/hadoop-metrics.properties
index d04dffc438..d04dffc438 100644
--- a/third_party/hadoop-0.20.0/conf/hadoop-metrics.properties
+++ b/core/lib/hadoop-0.20.0/conf/hadoop-metrics.properties
diff --git a/third_party/hadoop-0.20.0/conf/hadoop-policy.xml b/core/lib/hadoop-0.20.0/conf/hadoop-policy.xml
index ef48f2bbed..ef48f2bbed 100644
--- a/third_party/hadoop-0.20.0/conf/hadoop-policy.xml
+++ b/core/lib/hadoop-0.20.0/conf/hadoop-policy.xml
diff --git a/third_party/hadoop-0.20.0/conf/hdfs-site.xml b/core/lib/hadoop-0.20.0/conf/hdfs-site.xml
index 970c8fe0e8..970c8fe0e8 100644
--- a/third_party/hadoop-0.20.0/conf/hdfs-site.xml
+++ b/core/lib/hadoop-0.20.0/conf/hdfs-site.xml
diff --git a/third_party/hadoop-0.20.0/conf/log4j.properties b/core/lib/hadoop-0.20.0/conf/log4j.properties
index d797df6dab..d797df6dab 100644
--- a/third_party/hadoop-0.20.0/conf/log4j.properties
+++ b/core/lib/hadoop-0.20.0/conf/log4j.properties
diff --git a/third_party/hadoop-0.20.0/conf/mapred-site.xml b/core/lib/hadoop-0.20.0/conf/mapred-site.xml
index 970c8fe0e8..970c8fe0e8 100644
--- a/third_party/hadoop-0.20.0/conf/mapred-site.xml
+++ b/core/lib/hadoop-0.20.0/conf/mapred-site.xml
diff --git a/third_party/hadoop-0.20.0/conf/masters b/core/lib/hadoop-0.20.0/conf/masters
index 2fbb50c4a8..2fbb50c4a8 100644
--- a/third_party/hadoop-0.20.0/conf/masters
+++ b/core/lib/hadoop-0.20.0/conf/masters
diff --git a/third_party/hadoop-0.20.0/conf/slaves b/core/lib/hadoop-0.20.0/conf/slaves
index 2fbb50c4a8..2fbb50c4a8 100644
--- a/third_party/hadoop-0.20.0/conf/slaves
+++ b/core/lib/hadoop-0.20.0/conf/slaves
diff --git a/third_party/hadoop-0.20.0/conf/ssl-client.xml.example b/core/lib/hadoop-0.20.0/conf/ssl-client.xml.example
index ec3fd41fa8..ec3fd41fa8 100644
--- a/third_party/hadoop-0.20.0/conf/ssl-client.xml.example
+++ b/core/lib/hadoop-0.20.0/conf/ssl-client.xml.example
diff --git a/third_party/hadoop-0.20.0/conf/ssl-server.xml.example b/core/lib/hadoop-0.20.0/conf/ssl-server.xml.example
index 22e9cb0ebb..22e9cb0ebb 100644
--- a/third_party/hadoop-0.20.0/conf/ssl-server.xml.example
+++ b/core/lib/hadoop-0.20.0/conf/ssl-server.xml.example
diff --git a/third_party/hadoop-0.20.0/contrib/capacity-scheduler/hadoop-0.20.0-capacity-scheduler.jar b/core/lib/hadoop-0.20.0/contrib/capacity-scheduler/hadoop-0.20.0-capacity-scheduler.jar
index b4900e565e..b4900e565e 100644
--- a/third_party/hadoop-0.20.0/contrib/capacity-scheduler/hadoop-0.20.0-capacity-scheduler.jar
+++ b/core/lib/hadoop-0.20.0/contrib/capacity-scheduler/hadoop-0.20.0-capacity-scheduler.jar
diff --git a/third_party/hadoop-0.20.0/contrib/datajoin/hadoop-0.20.0-datajoin.jar b/core/lib/hadoop-0.20.0/contrib/datajoin/hadoop-0.20.0-datajoin.jar
index 21294d4d1d..21294d4d1d 100644
--- a/third_party/hadoop-0.20.0/contrib/datajoin/hadoop-0.20.0-datajoin.jar
+++ b/core/lib/hadoop-0.20.0/contrib/datajoin/hadoop-0.20.0-datajoin.jar
diff --git a/third_party/hadoop-0.20.0/contrib/eclipse-plugin/hadoop-0.20.0-eclipse-plugin.jar b/core/lib/hadoop-0.20.0/contrib/eclipse-plugin/hadoop-0.20.0-eclipse-plugin.jar
index 7b316393f6..7b316393f6 100644
--- a/third_party/hadoop-0.20.0/contrib/eclipse-plugin/hadoop-0.20.0-eclipse-plugin.jar
+++ b/core/lib/hadoop-0.20.0/contrib/eclipse-plugin/hadoop-0.20.0-eclipse-plugin.jar
diff --git a/third_party/hadoop-0.20.0/contrib/fairscheduler/hadoop-0.20.0-fairscheduler.jar b/core/lib/hadoop-0.20.0/contrib/fairscheduler/hadoop-0.20.0-fairscheduler.jar
index 758b98367c..758b98367c 100644
--- a/third_party/hadoop-0.20.0/contrib/fairscheduler/hadoop-0.20.0-fairscheduler.jar
+++ b/core/lib/hadoop-0.20.0/contrib/fairscheduler/hadoop-0.20.0-fairscheduler.jar
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/README b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/README
index 2c33988926..2c33988926 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/README
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/README
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy
index 1b1e597891..1b1e597891 100755
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-config.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-config.sh
index 8fe6aac68b..8fe6aac68b 100755
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-config.sh
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-config.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemon.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemon.sh
index 6d5a75247f..6d5a75247f 100755
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemon.sh
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemon.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemons.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemons.sh
index 7dd8568a3b..7dd8568a3b 100755
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemons.sh
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemons.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-slaves.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-slaves.sh
index db54bd5b38..db54bd5b38 100755
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-slaves.sh
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-slaves.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/start-hdfsproxy.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/start-hdfsproxy.sh
index 2592d9c8cc..2592d9c8cc 100755
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/start-hdfsproxy.sh
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/start-hdfsproxy.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/stop-hdfsproxy.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/stop-hdfsproxy.sh
index 78089e31cf..78089e31cf 100755
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/stop-hdfsproxy.sh
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/stop-hdfsproxy.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/build.xml b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/build.xml
index e62b2f279a..e62b2f279a 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/build.xml
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/build.xml
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/configuration.xsl b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/configuration.xsl
index 377cdbeb93..377cdbeb93 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/configuration.xsl
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/configuration.xsl
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-default.xml b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-default.xml
index 0d2a006c8e..0d2a006c8e 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-default.xml
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-default.xml
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh
index a0ff7a5d27..a0ff7a5d27 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh.template b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh.template
index a0ff7a5d27..a0ff7a5d27 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh.template
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh.template
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-hosts b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-hosts
index 2fbb50c4a8..2fbb50c4a8 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-hosts
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-hosts
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/log4j.properties b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/log4j.properties
index 2520ab3795..2520ab3795 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/log4j.properties
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/log4j.properties
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/user-certs.xml b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/user-certs.xml
index f572a55294..f572a55294 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/user-certs.xml
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/user-certs.xml
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/user-permissions.xml b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/user-permissions.xml
index b7373751bd..b7373751bd 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/user-permissions.xml
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/user-permissions.xml
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/hdfsproxy-1.0.jar b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/hdfsproxy-1.0.jar
index a313391dfb..a313391dfb 100644
--- a/third_party/hadoop-0.20.0/contrib/hdfsproxy/hdfsproxy-1.0.jar
+++ b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/hdfsproxy-1.0.jar
diff --git a/third_party/hadoop-0.20.0/contrib/index/hadoop-0.20.0-index.jar b/core/lib/hadoop-0.20.0/contrib/index/hadoop-0.20.0-index.jar
index f1f850fcd3..f1f850fcd3 100644
--- a/third_party/hadoop-0.20.0/contrib/index/hadoop-0.20.0-index.jar
+++ b/core/lib/hadoop-0.20.0/contrib/index/hadoop-0.20.0-index.jar
diff --git a/third_party/hadoop-0.20.0/contrib/streaming/hadoop-0.20.0-streaming.jar b/core/lib/hadoop-0.20.0/contrib/streaming/hadoop-0.20.0-streaming.jar
index 84251e3a3c..84251e3a3c 100644
--- a/third_party/hadoop-0.20.0/contrib/streaming/hadoop-0.20.0-streaming.jar
+++ b/core/lib/hadoop-0.20.0/contrib/streaming/hadoop-0.20.0-streaming.jar
diff --git a/third_party/hadoop-0.20.0/contrib/thriftfs/hadoop-0.20.0-thriftfs.jar b/core/lib/hadoop-0.20.0/contrib/thriftfs/hadoop-0.20.0-thriftfs.jar
index bf10c05e1d..bf10c05e1d 100644
--- a/third_party/hadoop-0.20.0/contrib/thriftfs/hadoop-0.20.0-thriftfs.jar
+++ b/core/lib/hadoop-0.20.0/contrib/thriftfs/hadoop-0.20.0-thriftfs.jar
diff --git a/third_party/hadoop-0.20.0/contrib/vaidya/bin/vaidya.sh b/core/lib/hadoop-0.20.0/contrib/vaidya/bin/vaidya.sh
index ada6715342..ada6715342 100755
--- a/third_party/hadoop-0.20.0/contrib/vaidya/bin/vaidya.sh
+++ b/core/lib/hadoop-0.20.0/contrib/vaidya/bin/vaidya.sh
diff --git a/third_party/hadoop-0.20.0/contrib/vaidya/conf/postex_diagnosis_tests.xml b/core/lib/hadoop-0.20.0/contrib/vaidya/conf/postex_diagnosis_tests.xml
index f30d5d9cc8..f30d5d9cc8 100644
--- a/third_party/hadoop-0.20.0/contrib/vaidya/conf/postex_diagnosis_tests.xml
+++ b/core/lib/hadoop-0.20.0/contrib/vaidya/conf/postex_diagnosis_tests.xml
diff --git a/third_party/hadoop-0.20.0/contrib/vaidya/hadoop-0.20.0-vaidya.jar b/core/lib/hadoop-0.20.0/contrib/vaidya/hadoop-0.20.0-vaidya.jar
index 534b18d974..534b18d974 100644
--- a/third_party/hadoop-0.20.0/contrib/vaidya/hadoop-0.20.0-vaidya.jar
+++ b/core/lib/hadoop-0.20.0/contrib/vaidya/hadoop-0.20.0-vaidya.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-ant.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-ant.jar
index 0c2b74e74f..0c2b74e74f 100644
--- a/third_party/hadoop-0.20.0/hadoop-0.20.0-ant.jar
+++ b/core/lib/hadoop-0.20.0/hadoop-0.20.0-ant.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-core.jar
index c99ce6d44d..c99ce6d44d 100644
--- a/third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar
+++ b/core/lib/hadoop-0.20.0/hadoop-0.20.0-core.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-examples.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-examples.jar
index 23b88f885d..23b88f885d 100644
--- a/third_party/hadoop-0.20.0/hadoop-0.20.0-examples.jar
+++ b/core/lib/hadoop-0.20.0/hadoop-0.20.0-examples.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-test.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-test.jar
index 02b17d4160..02b17d4160 100644
--- a/third_party/hadoop-0.20.0/hadoop-0.20.0-test.jar
+++ b/core/lib/hadoop-0.20.0/hadoop-0.20.0-test.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-tools.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-tools.jar
index 60f5e600a6..60f5e600a6 100644
--- a/third_party/hadoop-0.20.0/hadoop-0.20.0-tools.jar
+++ b/core/lib/hadoop-0.20.0/hadoop-0.20.0-tools.jar
diff --git a/third_party/hadoop-0.20.0/ivy.xml b/core/lib/hadoop-0.20.0/ivy.xml
index 051ac6efb0..051ac6efb0 100644
--- a/third_party/hadoop-0.20.0/ivy.xml
+++ b/core/lib/hadoop-0.20.0/ivy.xml
diff --git a/third_party/hadoop-0.20.0/ivy/hadoop-core.pom b/core/lib/hadoop-0.20.0/ivy/hadoop-core.pom
index ffdd18951e..ffdd18951e 100644
--- a/third_party/hadoop-0.20.0/ivy/hadoop-core.pom
+++ b/core/lib/hadoop-0.20.0/ivy/hadoop-core.pom
diff --git a/third_party/hadoop-0.20.0/ivy/ivy-2.0.0-rc2.jar b/core/lib/hadoop-0.20.0/ivy/ivy-2.0.0-rc2.jar
index fa9ef21c7b..fa9ef21c7b 100644
--- a/third_party/hadoop-0.20.0/ivy/ivy-2.0.0-rc2.jar
+++ b/core/lib/hadoop-0.20.0/ivy/ivy-2.0.0-rc2.jar
diff --git a/third_party/hadoop-0.20.0/ivy/ivysettings.xml b/core/lib/hadoop-0.20.0/ivy/ivysettings.xml
index a7fcd22031..a7fcd22031 100644
--- a/third_party/hadoop-0.20.0/ivy/ivysettings.xml
+++ b/core/lib/hadoop-0.20.0/ivy/ivysettings.xml
diff --git a/third_party/hadoop-0.20.0/ivy/libraries.properties b/core/lib/hadoop-0.20.0/ivy/libraries.properties
index 17cf390d92..17cf390d92 100644
--- a/third_party/hadoop-0.20.0/ivy/libraries.properties
+++ b/core/lib/hadoop-0.20.0/ivy/libraries.properties
diff --git a/third_party/hadoop-0.20.0/lib/.DS_Store b/core/lib/hadoop-0.20.0/lib/.DS_Store
index e0d363a012..e0d363a012 100644
--- a/third_party/hadoop-0.20.0/lib/.DS_Store
+++ b/core/lib/hadoop-0.20.0/lib/.DS_Store
diff --git a/third_party/hadoop-0.20.0/lib/commons-cli-2.0-SNAPSHOT.jar b/core/lib/hadoop-0.20.0/lib/commons-cli-2.0-SNAPSHOT.jar
index 0b1d51072a..0b1d51072a 100644
--- a/third_party/hadoop-0.20.0/lib/commons-cli-2.0-SNAPSHOT.jar
+++ b/core/lib/hadoop-0.20.0/lib/commons-cli-2.0-SNAPSHOT.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-codec-1.3.jar b/core/lib/hadoop-0.20.0/lib/commons-codec-1.3.jar
index 957b6752af..957b6752af 100644
--- a/third_party/hadoop-0.20.0/lib/commons-codec-1.3.jar
+++ b/core/lib/hadoop-0.20.0/lib/commons-codec-1.3.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-el-1.0.jar b/core/lib/hadoop-0.20.0/lib/commons-el-1.0.jar
index 608ed796ca..608ed796ca 100644
--- a/third_party/hadoop-0.20.0/lib/commons-el-1.0.jar
+++ b/core/lib/hadoop-0.20.0/lib/commons-el-1.0.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-httpclient-3.0.1.jar b/core/lib/hadoop-0.20.0/lib/commons-httpclient-3.0.1.jar
index cfc777c71d..cfc777c71d 100644
--- a/third_party/hadoop-0.20.0/lib/commons-httpclient-3.0.1.jar
+++ b/core/lib/hadoop-0.20.0/lib/commons-httpclient-3.0.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-logging-1.0.4.jar b/core/lib/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
index b73a80fab6..b73a80fab6 100644
--- a/third_party/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
+++ b/core/lib/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-logging-api-1.0.4.jar b/core/lib/hadoop-0.20.0/lib/commons-logging-api-1.0.4.jar
index ade9a13c78..ade9a13c78 100644
--- a/third_party/hadoop-0.20.0/lib/commons-logging-api-1.0.4.jar
+++ b/core/lib/hadoop-0.20.0/lib/commons-logging-api-1.0.4.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-net-1.4.1.jar b/core/lib/hadoop-0.20.0/lib/commons-net-1.4.1.jar
index 9666a92c80..9666a92c80 100644
--- a/third_party/hadoop-0.20.0/lib/commons-net-1.4.1.jar
+++ b/core/lib/hadoop-0.20.0/lib/commons-net-1.4.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/core-3.1.1.jar b/core/lib/hadoop-0.20.0/lib/core-3.1.1.jar
index ae0b635867..ae0b635867 100644
--- a/third_party/hadoop-0.20.0/lib/core-3.1.1.jar
+++ b/core/lib/hadoop-0.20.0/lib/core-3.1.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/hsqldb-1.8.0.10.LICENSE.txt b/core/lib/hadoop-0.20.0/lib/hsqldb-1.8.0.10.LICENSE.txt
index d45b9f8cc0..d45b9f8cc0 100644
--- a/third_party/hadoop-0.20.0/lib/hsqldb-1.8.0.10.LICENSE.txt
+++ b/core/lib/hadoop-0.20.0/lib/hsqldb-1.8.0.10.LICENSE.txt
diff --git a/third_party/hadoop-0.20.0/lib/hsqldb-1.8.0.10.jar b/core/lib/hadoop-0.20.0/lib/hsqldb-1.8.0.10.jar
index e010269ddf..e010269ddf 100644
--- a/third_party/hadoop-0.20.0/lib/hsqldb-1.8.0.10.jar
+++ b/core/lib/hadoop-0.20.0/lib/hsqldb-1.8.0.10.jar
diff --git a/third_party/hadoop-0.20.0/lib/jasper-compiler-5.5.12.jar b/core/lib/hadoop-0.20.0/lib/jasper-compiler-5.5.12.jar
index 2a410b4b58..2a410b4b58 100644
--- a/third_party/hadoop-0.20.0/lib/jasper-compiler-5.5.12.jar
+++ b/core/lib/hadoop-0.20.0/lib/jasper-compiler-5.5.12.jar
diff --git a/third_party/hadoop-0.20.0/lib/jasper-runtime-5.5.12.jar b/core/lib/hadoop-0.20.0/lib/jasper-runtime-5.5.12.jar
index 743d906c1f..743d906c1f 100644
--- a/third_party/hadoop-0.20.0/lib/jasper-runtime-5.5.12.jar
+++ b/core/lib/hadoop-0.20.0/lib/jasper-runtime-5.5.12.jar
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.17.0.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.17.0.xml
index 69dded3140..69dded3140 100644
--- a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.17.0.xml
+++ b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.17.0.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.1.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.1.xml
index fd844cbed0..fd844cbed0 100644
--- a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.1.xml
+++ b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.1.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.2.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.2.xml
index 08173ab82d..08173ab82d 100644
--- a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.2.xml
+++ b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.2.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.3.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.3.xml
index 564916fef7..564916fef7 100644
--- a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.3.xml
+++ b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.3.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.19.0.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.19.0.xml
index 557ac3cc59..557ac3cc59 100644
--- a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.19.0.xml
+++ b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.19.0.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.19.1.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.19.1.xml
index 92bdd2c799..92bdd2c799 100644
--- a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.19.1.xml
+++ b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.19.1.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.20.0.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.20.0.xml
index 9067cf1158..9067cf1158 100644
--- a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.20.0.xml
+++ b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.20.0.xml
diff --git a/third_party/hadoop-0.20.0/lib/jets3t-0.6.1.jar b/core/lib/hadoop-0.20.0/lib/jets3t-0.6.1.jar
index e4048dd685..e4048dd685 100644
--- a/third_party/hadoop-0.20.0/lib/jets3t-0.6.1.jar
+++ b/core/lib/hadoop-0.20.0/lib/jets3t-0.6.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/jetty-6.1.14.jar b/core/lib/hadoop-0.20.0/lib/jetty-6.1.14.jar
index 8c503bea21..8c503bea21 100644
--- a/third_party/hadoop-0.20.0/lib/jetty-6.1.14.jar
+++ b/core/lib/hadoop-0.20.0/lib/jetty-6.1.14.jar
diff --git a/third_party/hadoop-0.20.0/lib/jetty-util-6.1.14.jar b/core/lib/hadoop-0.20.0/lib/jetty-util-6.1.14.jar
index 8f924bb147..8f924bb147 100644
--- a/third_party/hadoop-0.20.0/lib/jetty-util-6.1.14.jar
+++ b/core/lib/hadoop-0.20.0/lib/jetty-util-6.1.14.jar
diff --git a/third_party/hadoop-0.20.0/lib/jsp-2.1/jsp-2.1.jar b/core/lib/hadoop-0.20.0/lib/jsp-2.1/jsp-2.1.jar
index bfdb566c13..bfdb566c13 100644
--- a/third_party/hadoop-0.20.0/lib/jsp-2.1/jsp-2.1.jar
+++ b/core/lib/hadoop-0.20.0/lib/jsp-2.1/jsp-2.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/jsp-2.1/jsp-api-2.1.jar b/core/lib/hadoop-0.20.0/lib/jsp-2.1/jsp-api-2.1.jar
index ac3a7a8f7e..ac3a7a8f7e 100644
--- a/third_party/hadoop-0.20.0/lib/jsp-2.1/jsp-api-2.1.jar
+++ b/core/lib/hadoop-0.20.0/lib/jsp-2.1/jsp-api-2.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/junit-3.8.1.jar b/core/lib/hadoop-0.20.0/lib/junit-3.8.1.jar
index 674d71e89e..674d71e89e 100644
--- a/third_party/hadoop-0.20.0/lib/junit-3.8.1.jar
+++ b/core/lib/hadoop-0.20.0/lib/junit-3.8.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/kfs-0.2.2.jar b/core/lib/hadoop-0.20.0/lib/kfs-0.2.2.jar
index aa32e74baf..aa32e74baf 100644
--- a/third_party/hadoop-0.20.0/lib/kfs-0.2.2.jar
+++ b/core/lib/hadoop-0.20.0/lib/kfs-0.2.2.jar
diff --git a/third_party/hadoop-0.20.0/lib/kfs-0.2.LICENSE.txt b/core/lib/hadoop-0.20.0/lib/kfs-0.2.LICENSE.txt
index d645695673..d645695673 100644
--- a/third_party/hadoop-0.20.0/lib/kfs-0.2.LICENSE.txt
+++ b/core/lib/hadoop-0.20.0/lib/kfs-0.2.LICENSE.txt
diff --git a/third_party/hadoop-0.20.0/lib/log4j-1.2.15.jar b/core/lib/hadoop-0.20.0/lib/log4j-1.2.15.jar
index c930a6ab4d..c930a6ab4d 100644
--- a/third_party/hadoop-0.20.0/lib/log4j-1.2.15.jar
+++ b/core/lib/hadoop-0.20.0/lib/log4j-1.2.15.jar
diff --git a/third_party/hadoop-0.20.0/lib/native/.DS_Store b/core/lib/hadoop-0.20.0/lib/native/.DS_Store
index ff86c6f7f2..ff86c6f7f2 100644
--- a/third_party/hadoop-0.20.0/lib/native/.DS_Store
+++ b/core/lib/hadoop-0.20.0/lib/native/.DS_Store
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.a b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.a
index d8d90cf067..d8d90cf067 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.a
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.a
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.la b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.la
index 2e772fb4ae..2e772fb4ae 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.la
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.la
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so
index fb2cbad0b5..fb2cbad0b5 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1 b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1
index fb2cbad0b5..fb2cbad0b5 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1.0.0 b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
index fb2cbad0b5..fb2cbad0b5 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.a b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.a
index 068d2d6bf6..068d2d6bf6 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.a
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.a
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.la b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.la
index 14941670b4..14941670b4 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.la
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.la
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so
index e3acc2b220..e3acc2b220 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1 b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1
index e3acc2b220..e3acc2b220 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1.0.0 b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1.0.0
index e3acc2b220..e3acc2b220 100644
--- a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1.0.0
+++ b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1.0.0
diff --git a/third_party/hadoop-0.20.0/lib/oro-2.0.8.jar b/core/lib/hadoop-0.20.0/lib/oro-2.0.8.jar
index 23488d2600..23488d2600 100644
--- a/third_party/hadoop-0.20.0/lib/oro-2.0.8.jar
+++ b/core/lib/hadoop-0.20.0/lib/oro-2.0.8.jar
diff --git a/third_party/hadoop-0.20.0/lib/servlet-api-2.5-6.1.14.jar b/core/lib/hadoop-0.20.0/lib/servlet-api-2.5-6.1.14.jar
index 6d7404fb72..6d7404fb72 100644
--- a/third_party/hadoop-0.20.0/lib/servlet-api-2.5-6.1.14.jar
+++ b/core/lib/hadoop-0.20.0/lib/servlet-api-2.5-6.1.14.jar
diff --git a/third_party/hadoop-0.20.0/lib/xmlenc-0.52.jar b/core/lib/hadoop-0.20.0/lib/xmlenc-0.52.jar
index ec568b4c9e..ec568b4c9e 100644
--- a/third_party/hadoop-0.20.0/lib/xmlenc-0.52.jar
+++ b/core/lib/hadoop-0.20.0/lib/xmlenc-0.52.jar
diff --git a/third_party/hadoop-0.20.0/librecordio/librecordio.a b/core/lib/hadoop-0.20.0/librecordio/librecordio.a
index 49f7c22d1e..49f7c22d1e 100644
--- a/third_party/hadoop-0.20.0/librecordio/librecordio.a
+++ b/core/lib/hadoop-0.20.0/librecordio/librecordio.a
diff --git a/third_party/hadoop-0.20.0/webapps/datanode/WEB-INF/web.xml b/core/lib/hadoop-0.20.0/webapps/datanode/WEB-INF/web.xml
index c271b62815..c271b62815 100644
--- a/third_party/hadoop-0.20.0/webapps/datanode/WEB-INF/web.xml
+++ b/core/lib/hadoop-0.20.0/webapps/datanode/WEB-INF/web.xml
diff --git a/third_party/hadoop-0.20.0/webapps/hdfs/WEB-INF/web.xml b/core/lib/hadoop-0.20.0/webapps/hdfs/WEB-INF/web.xml
index 40a73d5293..40a73d5293 100644
--- a/third_party/hadoop-0.20.0/webapps/hdfs/WEB-INF/web.xml
+++ b/core/lib/hadoop-0.20.0/webapps/hdfs/WEB-INF/web.xml
diff --git a/third_party/hadoop-0.20.0/webapps/hdfs/index.html b/core/lib/hadoop-0.20.0/webapps/hdfs/index.html
index b9ad74218a..b9ad74218a 100644
--- a/third_party/hadoop-0.20.0/webapps/hdfs/index.html
+++ b/core/lib/hadoop-0.20.0/webapps/hdfs/index.html
diff --git a/third_party/hadoop-0.20.0/webapps/job/WEB-INF/web.xml b/core/lib/hadoop-0.20.0/webapps/job/WEB-INF/web.xml
index e124ff744b..e124ff744b 100644
--- a/third_party/hadoop-0.20.0/webapps/job/WEB-INF/web.xml
+++ b/core/lib/hadoop-0.20.0/webapps/job/WEB-INF/web.xml
diff --git a/third_party/hadoop-0.20.0/webapps/job/index.html b/core/lib/hadoop-0.20.0/webapps/job/index.html
index 0cc16dce3d..0cc16dce3d 100644
--- a/third_party/hadoop-0.20.0/webapps/job/index.html
+++ b/core/lib/hadoop-0.20.0/webapps/job/index.html
diff --git a/third_party/hadoop-0.20.0/webapps/static/hadoop-logo.jpg b/core/lib/hadoop-0.20.0/webapps/static/hadoop-logo.jpg
index 809525d9f1..809525d9f1 100644
--- a/third_party/hadoop-0.20.0/webapps/static/hadoop-logo.jpg
+++ b/core/lib/hadoop-0.20.0/webapps/static/hadoop-logo.jpg
diff --git a/third_party/hadoop-0.20.0/webapps/static/hadoop.css b/core/lib/hadoop-0.20.0/webapps/static/hadoop.css
index 0560cb3075..0560cb3075 100644
--- a/third_party/hadoop-0.20.0/webapps/static/hadoop.css
+++ b/core/lib/hadoop-0.20.0/webapps/static/hadoop.css
diff --git a/third_party/hadoop-0.20.0/webapps/static/jobconf.xsl b/core/lib/hadoop-0.20.0/webapps/static/jobconf.xsl
index f3c2e33cef..f3c2e33cef 100644
--- a/third_party/hadoop-0.20.0/webapps/static/jobconf.xsl
+++ b/core/lib/hadoop-0.20.0/webapps/static/jobconf.xsl
diff --git a/third_party/hadoop-0.20.0/webapps/static/jobtracker.js b/core/lib/hadoop-0.20.0/webapps/static/jobtracker.js
index 7da16c1fcc..7da16c1fcc 100644
--- a/third_party/hadoop-0.20.0/webapps/static/jobtracker.js
+++ b/core/lib/hadoop-0.20.0/webapps/static/jobtracker.js
diff --git a/third_party/hadoop-0.20.0/webapps/task/WEB-INF/web.xml b/core/lib/hadoop-0.20.0/webapps/task/WEB-INF/web.xml
index 44a03bed6b..44a03bed6b 100644
--- a/third_party/hadoop-0.20.0/webapps/task/WEB-INF/web.xml
+++ b/core/lib/hadoop-0.20.0/webapps/task/WEB-INF/web.xml
diff --git a/third_party/hadoop-0.20.0/webapps/task/index.html b/core/lib/hadoop-0.20.0/webapps/task/index.html
index ab3d56ba46..ab3d56ba46 100644
--- a/third_party/hadoop-0.20.0/webapps/task/index.html
+++ b/core/lib/hadoop-0.20.0/webapps/task/index.html
diff --git a/third_party/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar b/core/lib/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
index d9ef50be6d..d9ef50be6d 100644
--- a/third_party/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
+++ b/core/lib/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
diff --git a/third_party/jetty-7.1.6.v20100715/servlet-api-2.5.jar b/core/lib/jetty-7.1.6.v20100715/servlet-api-2.5.jar
index fb52493468..fb52493468 100644
--- a/third_party/jetty-7.1.6.v20100715/servlet-api-2.5.jar
+++ b/core/lib/jetty-7.1.6.v20100715/servlet-api-2.5.jar
diff --git a/core/lib/jline.jar b/core/lib/jline.jar
new file mode 100644
index 0000000000..6ed67faab6
--- /dev/null
+++ b/core/lib/jline.jar
diff --git a/third_party/liblzf-3.5/Changes b/core/lib/liblzf-3.5/Changes
index 09581ff20f..09581ff20f 100644
--- a/third_party/liblzf-3.5/Changes
+++ b/core/lib/liblzf-3.5/Changes
diff --git a/third_party/liblzf-3.5/LICENSE b/core/lib/liblzf-3.5/LICENSE
index 9d8e6ec9c8..9d8e6ec9c8 100644
--- a/third_party/liblzf-3.5/LICENSE
+++ b/core/lib/liblzf-3.5/LICENSE
diff --git a/third_party/liblzf-3.5/Makefile b/core/lib/liblzf-3.5/Makefile
index 2af118f9d3..2af118f9d3 100644
--- a/third_party/liblzf-3.5/Makefile
+++ b/core/lib/liblzf-3.5/Makefile
diff --git a/third_party/liblzf-3.5/Makefile.in b/core/lib/liblzf-3.5/Makefile.in
index bbd53c0370..bbd53c0370 100644
--- a/third_party/liblzf-3.5/Makefile.in
+++ b/core/lib/liblzf-3.5/Makefile.in
diff --git a/third_party/liblzf-3.5/README b/core/lib/liblzf-3.5/README
index b5379a3892..b5379a3892 100644
--- a/third_party/liblzf-3.5/README
+++ b/core/lib/liblzf-3.5/README
diff --git a/third_party/liblzf-3.5/config.h b/core/lib/liblzf-3.5/config.h
index 966d01ff7d..966d01ff7d 100644
--- a/third_party/liblzf-3.5/config.h
+++ b/core/lib/liblzf-3.5/config.h
diff --git a/third_party/liblzf-3.5/config.h.in b/core/lib/liblzf-3.5/config.h.in
index 5fd69c6bd5..5fd69c6bd5 100644
--- a/third_party/liblzf-3.5/config.h.in
+++ b/core/lib/liblzf-3.5/config.h.in
diff --git a/third_party/liblzf-3.5/config.log b/core/lib/liblzf-3.5/config.log
index dfa0a83845..dfa0a83845 100644
--- a/third_party/liblzf-3.5/config.log
+++ b/core/lib/liblzf-3.5/config.log
diff --git a/third_party/liblzf-3.5/config.status b/core/lib/liblzf-3.5/config.status
index 2498ff6649..2498ff6649 100755
--- a/third_party/liblzf-3.5/config.status
+++ b/core/lib/liblzf-3.5/config.status
diff --git a/third_party/liblzf-3.5/configure b/core/lib/liblzf-3.5/configure
index 7a3a2b25c0..7a3a2b25c0 100755
--- a/third_party/liblzf-3.5/configure
+++ b/core/lib/liblzf-3.5/configure
diff --git a/third_party/liblzf-3.5/configure.ac b/core/lib/liblzf-3.5/configure.ac
index 58316a01b3..58316a01b3 100644
--- a/third_party/liblzf-3.5/configure.ac
+++ b/core/lib/liblzf-3.5/configure.ac
diff --git a/third_party/liblzf-3.5/crc32.h b/core/lib/liblzf-3.5/crc32.h
index cf8f6d4097..cf8f6d4097 100644
--- a/third_party/liblzf-3.5/crc32.h
+++ b/core/lib/liblzf-3.5/crc32.h
diff --git a/third_party/liblzf-3.5/cs/CLZF.cs b/core/lib/liblzf-3.5/cs/CLZF.cs
index 4e1b5d50b8..4e1b5d50b8 100644
--- a/third_party/liblzf-3.5/cs/CLZF.cs
+++ b/core/lib/liblzf-3.5/cs/CLZF.cs
diff --git a/third_party/liblzf-3.5/cs/README b/core/lib/liblzf-3.5/cs/README
index 9964aff103..9964aff103 100644
--- a/third_party/liblzf-3.5/cs/README
+++ b/core/lib/liblzf-3.5/cs/README
diff --git a/third_party/liblzf-3.5/install-sh b/core/lib/liblzf-3.5/install-sh
index e9de23842d..e9de23842d 100755
--- a/third_party/liblzf-3.5/install-sh
+++ b/core/lib/liblzf-3.5/install-sh
diff --git a/third_party/liblzf-3.5/lzf.c b/core/lib/liblzf-3.5/lzf.c
index bedfdb6fea..bedfdb6fea 100644
--- a/third_party/liblzf-3.5/lzf.c
+++ b/core/lib/liblzf-3.5/lzf.c
diff --git a/third_party/liblzf-3.5/lzf.h b/core/lib/liblzf-3.5/lzf.h
index 919b6e6be2..919b6e6be2 100644
--- a/third_party/liblzf-3.5/lzf.h
+++ b/core/lib/liblzf-3.5/lzf.h
diff --git a/third_party/liblzf-3.5/lzfP.h b/core/lib/liblzf-3.5/lzfP.h
index d533f18292..d533f18292 100644
--- a/third_party/liblzf-3.5/lzfP.h
+++ b/core/lib/liblzf-3.5/lzfP.h
diff --git a/third_party/liblzf-3.5/lzf_c.c b/core/lib/liblzf-3.5/lzf_c.c
index 0b5d816dd0..0b5d816dd0 100644
--- a/third_party/liblzf-3.5/lzf_c.c
+++ b/core/lib/liblzf-3.5/lzf_c.c
diff --git a/third_party/liblzf-3.5/lzf_d.c b/core/lib/liblzf-3.5/lzf_d.c
index 9e2cd829c3..9e2cd829c3 100644
--- a/third_party/liblzf-3.5/lzf_d.c
+++ b/core/lib/liblzf-3.5/lzf_d.c
diff --git a/third_party/mesos.jar b/core/lib/mesos.jar
index 60d299c8af..60d299c8af 100644
--- a/third_party/mesos.jar
+++ b/core/lib/mesos.jar
diff --git a/third_party/scalacheck_2.8.0-1.7.jar b/core/lib/scalacheck_2.8.0-1.7.jar
index fb3c0e9e12..fb3c0e9e12 100644
--- a/third_party/scalacheck_2.8.0-1.7.jar
+++ b/core/lib/scalacheck_2.8.0-1.7.jar
diff --git a/third_party/scalatest-1.2/LICENSE b/core/lib/scalatest-1.2/LICENSE
index d645695673..d645695673 100644
--- a/third_party/scalatest-1.2/LICENSE
+++ b/core/lib/scalatest-1.2/LICENSE
diff --git a/third_party/scalatest-1.2/NOTICE b/core/lib/scalatest-1.2/NOTICE
index a405cbd58a..a405cbd58a 100644
--- a/third_party/scalatest-1.2/NOTICE
+++ b/core/lib/scalatest-1.2/NOTICE
diff --git a/third_party/scalatest-1.2/README.txt b/core/lib/scalatest-1.2/README.txt
index d505b9c640..d505b9c640 100644
--- a/third_party/scalatest-1.2/README.txt
+++ b/core/lib/scalatest-1.2/README.txt
diff --git a/third_party/scalatest-1.2/scalatest-1.2.jar b/core/lib/scalatest-1.2/scalatest-1.2.jar
index cb8db9bdf5..cb8db9bdf5 100644
--- a/third_party/scalatest-1.2/scalatest-1.2.jar
+++ b/core/lib/scalatest-1.2/scalatest-1.2.jar
diff --git a/third_party/slf4j-1.6.1/slf4j-api-1.6.1.jar b/core/lib/slf4j-1.6.1/slf4j-api-1.6.1.jar
index 42e0ad0de7..42e0ad0de7 100644
--- a/third_party/slf4j-1.6.1/slf4j-api-1.6.1.jar
+++ b/core/lib/slf4j-1.6.1/slf4j-api-1.6.1.jar
diff --git a/third_party/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar b/core/lib/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
index 873d11983e..873d11983e 100644
--- a/third_party/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
+++ b/core/lib/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
diff --git a/src/scala/spark/Accumulators.scala b/core/src/main/scala/spark/Accumulators.scala
index ee93d3c85c..ee93d3c85c 100644
--- a/src/scala/spark/Accumulators.scala
+++ b/core/src/main/scala/spark/Accumulators.scala
diff --git a/src/scala/spark/BitTorrentBroadcast.scala b/core/src/main/scala/spark/BitTorrentBroadcast.scala
index 0ef0d92071..0ef0d92071 100644
--- a/src/scala/spark/BitTorrentBroadcast.scala
+++ b/core/src/main/scala/spark/BitTorrentBroadcast.scala
diff --git a/src/scala/spark/BoundedMemoryCache.scala b/core/src/main/scala/spark/BoundedMemoryCache.scala
index 19d9bebfe5..19d9bebfe5 100644
--- a/src/scala/spark/BoundedMemoryCache.scala
+++ b/core/src/main/scala/spark/BoundedMemoryCache.scala
diff --git a/src/scala/spark/Broadcast.scala b/core/src/main/scala/spark/Broadcast.scala
index 485c6d2cbc..485c6d2cbc 100644
--- a/src/scala/spark/Broadcast.scala
+++ b/core/src/main/scala/spark/Broadcast.scala
diff --git a/src/scala/spark/Cache.scala b/core/src/main/scala/spark/Cache.scala
index 9887520758..9887520758 100644
--- a/src/scala/spark/Cache.scala
+++ b/core/src/main/scala/spark/Cache.scala
diff --git a/src/scala/spark/ChainedBroadcast.scala b/core/src/main/scala/spark/ChainedBroadcast.scala
index 5978bdea05..5978bdea05 100644
--- a/src/scala/spark/ChainedBroadcast.scala
+++ b/core/src/main/scala/spark/ChainedBroadcast.scala
diff --git a/src/scala/spark/ClosureCleaner.scala b/core/src/main/scala/spark/ClosureCleaner.scala
index 0e0b3954d4..0e0b3954d4 100644
--- a/src/scala/spark/ClosureCleaner.scala
+++ b/core/src/main/scala/spark/ClosureCleaner.scala
diff --git a/src/scala/spark/DfsBroadcast.scala b/core/src/main/scala/spark/DfsBroadcast.scala
index 480d6dd9b1..480d6dd9b1 100644
--- a/src/scala/spark/DfsBroadcast.scala
+++ b/core/src/main/scala/spark/DfsBroadcast.scala
diff --git a/src/scala/spark/DfsShuffle.scala b/core/src/main/scala/spark/DfsShuffle.scala
index 7a42bf2d06..7a42bf2d06 100644
--- a/src/scala/spark/DfsShuffle.scala
+++ b/core/src/main/scala/spark/DfsShuffle.scala
diff --git a/src/scala/spark/Executor.scala b/core/src/main/scala/spark/Executor.scala
index b4d023b428..b4d023b428 100644
--- a/src/scala/spark/Executor.scala
+++ b/core/src/main/scala/spark/Executor.scala
diff --git a/src/scala/spark/HadoopFile.scala b/core/src/main/scala/spark/HadoopFile.scala
index a63c9d8a94..a63c9d8a94 100644
--- a/src/scala/spark/HadoopFile.scala
+++ b/core/src/main/scala/spark/HadoopFile.scala
diff --git a/src/scala/spark/HttpServer.scala b/core/src/main/scala/spark/HttpServer.scala
index d2a663ac1f..d2a663ac1f 100644
--- a/src/scala/spark/HttpServer.scala
+++ b/core/src/main/scala/spark/HttpServer.scala
diff --git a/src/scala/spark/Job.scala b/core/src/main/scala/spark/Job.scala
index 6abbcbce51..6abbcbce51 100644
--- a/src/scala/spark/Job.scala
+++ b/core/src/main/scala/spark/Job.scala
diff --git a/src/scala/spark/LocalFileShuffle.scala b/core/src/main/scala/spark/LocalFileShuffle.scala
index 367599cfb4..367599cfb4 100644
--- a/src/scala/spark/LocalFileShuffle.scala
+++ b/core/src/main/scala/spark/LocalFileShuffle.scala
diff --git a/src/scala/spark/LocalScheduler.scala b/core/src/main/scala/spark/LocalScheduler.scala
index 20954a1224..20954a1224 100644
--- a/src/scala/spark/LocalScheduler.scala
+++ b/core/src/main/scala/spark/LocalScheduler.scala
diff --git a/src/scala/spark/Logging.scala b/core/src/main/scala/spark/Logging.scala
index 2d1feebbb1..2d1feebbb1 100644
--- a/src/scala/spark/Logging.scala
+++ b/core/src/main/scala/spark/Logging.scala
diff --git a/src/scala/spark/MesosScheduler.scala b/core/src/main/scala/spark/MesosScheduler.scala
index 6a592d13c3..6a592d13c3 100644
--- a/src/scala/spark/MesosScheduler.scala
+++ b/core/src/main/scala/spark/MesosScheduler.scala
diff --git a/src/scala/spark/NumberedSplitRDD.scala b/core/src/main/scala/spark/NumberedSplitRDD.scala
index 7b12210d84..7b12210d84 100644
--- a/src/scala/spark/NumberedSplitRDD.scala
+++ b/core/src/main/scala/spark/NumberedSplitRDD.scala
diff --git a/src/scala/spark/ParallelArray.scala b/core/src/main/scala/spark/ParallelArray.scala
index a01904d61c..a01904d61c 100644
--- a/src/scala/spark/ParallelArray.scala
+++ b/core/src/main/scala/spark/ParallelArray.scala
diff --git a/src/scala/spark/RDD.scala b/core/src/main/scala/spark/RDD.scala
index bac59319a0..39f2dc4458 100644
--- a/src/scala/spark/RDD.scala
+++ b/core/src/main/scala/spark/RDD.scala
@@ -359,7 +359,7 @@ extends RDD[Pair[T, U]](sc) {
   : RDD[(K, C)] =
   {
     val shufClass = Class.forName(System.getProperty(
-      "spark.shuffle.class", "spark.DfsShuffle"))
+      "spark.shuffle.class", "spark.LocalFileShuffle"))
     val shuf = shufClass.newInstance().asInstanceOf[Shuffle[K, V, C]]
     shuf.compute(self, numSplits, createCombiner, mergeValue, mergeCombiners)
   }
diff --git a/src/scala/spark/Scheduler.scala b/core/src/main/scala/spark/Scheduler.scala
index b9f3128c82..b9f3128c82 100644
--- a/src/scala/spark/Scheduler.scala
+++ b/core/src/main/scala/spark/Scheduler.scala
diff --git a/src/scala/spark/SerializableWritable.scala b/core/src/main/scala/spark/SerializableWritable.scala
index ae393d06d3..ae393d06d3 100644
--- a/src/scala/spark/SerializableWritable.scala
+++ b/core/src/main/scala/spark/SerializableWritable.scala
diff --git a/src/scala/spark/Shuffle.scala b/core/src/main/scala/spark/Shuffle.scala
index 4c5649b537..4c5649b537 100644
--- a/src/scala/spark/Shuffle.scala
+++ b/core/src/main/scala/spark/Shuffle.scala
diff --git a/src/scala/spark/SimpleJob.scala b/core/src/main/scala/spark/SimpleJob.scala
index 09846ccc34..09846ccc34 100644
--- a/src/scala/spark/SimpleJob.scala
+++ b/core/src/main/scala/spark/SimpleJob.scala
diff --git a/src/scala/spark/SizeEstimator.scala b/core/src/main/scala/spark/SizeEstimator.scala
index 12dd19d704..12dd19d704 100644
--- a/src/scala/spark/SizeEstimator.scala
+++ b/core/src/main/scala/spark/SizeEstimator.scala
diff --git a/src/scala/spark/SoftReferenceCache.scala b/core/src/main/scala/spark/SoftReferenceCache.scala
index e84aa57efa..e84aa57efa 100644
--- a/src/scala/spark/SoftReferenceCache.scala
+++ b/core/src/main/scala/spark/SoftReferenceCache.scala
diff --git a/src/scala/spark/SparkContext.scala b/core/src/main/scala/spark/SparkContext.scala
index 4ca66f5372..4ca66f5372 100644
--- a/src/scala/spark/SparkContext.scala
+++ b/core/src/main/scala/spark/SparkContext.scala
diff --git a/src/scala/spark/SparkException.scala b/core/src/main/scala/spark/SparkException.scala
index 6f9be1a94f..6f9be1a94f 100644
--- a/src/scala/spark/SparkException.scala
+++ b/core/src/main/scala/spark/SparkException.scala
diff --git a/src/scala/spark/Split.scala b/core/src/main/scala/spark/Split.scala
index 116cd16370..116cd16370 100644
--- a/src/scala/spark/Split.scala
+++ b/core/src/main/scala/spark/Split.scala
diff --git a/src/scala/spark/Task.scala b/core/src/main/scala/spark/Task.scala
index 6e94009f6e..6e94009f6e 100644
--- a/src/scala/spark/Task.scala
+++ b/core/src/main/scala/spark/Task.scala
diff --git a/src/scala/spark/TaskResult.scala b/core/src/main/scala/spark/TaskResult.scala
index db33c9ff44..db33c9ff44 100644
--- a/src/scala/spark/TaskResult.scala
+++ b/core/src/main/scala/spark/TaskResult.scala
diff --git a/src/scala/spark/Utils.scala b/core/src/main/scala/spark/Utils.scala
index e333dd9c91..e333dd9c91 100644
--- a/src/scala/spark/Utils.scala
+++ b/core/src/main/scala/spark/Utils.scala
diff --git a/src/scala/spark/WeakReferenceCache.scala b/core/src/main/scala/spark/WeakReferenceCache.scala
index ddca065454..ddca065454 100644
--- a/src/scala/spark/WeakReferenceCache.scala
+++ b/core/src/main/scala/spark/WeakReferenceCache.scala
diff --git a/src/scala/spark/repl/ExecutorClassLoader.scala b/core/src/main/scala/spark/repl/ExecutorClassLoader.scala
index 13d81ec1cf..13d81ec1cf 100644
--- a/src/scala/spark/repl/ExecutorClassLoader.scala
+++ b/core/src/main/scala/spark/repl/ExecutorClassLoader.scala
diff --git a/src/scala/spark/repl/Main.scala b/core/src/main/scala/spark/repl/Main.scala
index f00df5aa58..f00df5aa58 100644
--- a/src/scala/spark/repl/Main.scala
+++ b/core/src/main/scala/spark/repl/Main.scala
diff --git a/src/scala/spark/repl/SparkCompletion.scala b/core/src/main/scala/spark/repl/SparkCompletion.scala
index d67438445b..9fa41736f3 100644
--- a/src/scala/spark/repl/SparkCompletion.scala
+++ b/core/src/main/scala/spark/repl/SparkCompletion.scala
@@ -310,7 +310,7 @@ class SparkCompletion(val repl: SparkInterpreter) extends SparkCompletionOutput
       else xs.reduceLeft(_ zip _ takeWhile (x => x._1 == x._2) map (_._1) mkString)
 
     // This is jline's entry point for completion.
-    override def complete(_buf: String, cursor: Int, candidates: JList[String]): Int = {
+    override def complete(_buf: String, cursor: Int, candidates: java.util.List[java.lang.String]): Int = {
       val buf = onull(_buf)
       verbosity = if (isConsecutiveTabs(buf, cursor)) verbosity + 1 else 0
       DBG("complete(%s, %d) last = (%s, %d), verbosity: %s".format(buf, cursor, lastBuf, lastCursor, verbosity))
@@ -321,7 +321,7 @@ class SparkCompletion(val repl: SparkInterpreter) extends SparkCompletionOutput
           case Nil  => None
           case xs   =>
             // modify in place and return the position
-            xs foreach (candidates add _)
+            xs.foreach(x => candidates.add(x))
 
             // update the last buffer unless this is an alternatives list
             if (xs contains "") Some(p.cursor)
diff --git a/src/scala/spark/repl/SparkCompletionOutput.scala b/core/src/main/scala/spark/repl/SparkCompletionOutput.scala
index 5ac46e3412..5ac46e3412 100644
--- a/src/scala/spark/repl/SparkCompletionOutput.scala
+++ b/core/src/main/scala/spark/repl/SparkCompletionOutput.scala
diff --git a/src/scala/spark/repl/SparkInteractiveReader.scala b/core/src/main/scala/spark/repl/SparkInteractiveReader.scala
index 4f5a0a6fa0..4f5a0a6fa0 100644
--- a/src/scala/spark/repl/SparkInteractiveReader.scala
+++ b/core/src/main/scala/spark/repl/SparkInteractiveReader.scala
diff --git a/src/scala/spark/repl/SparkInterpreter.scala b/core/src/main/scala/spark/repl/SparkInterpreter.scala
index 10ea346658..10ea346658 100644
--- a/src/scala/spark/repl/SparkInterpreter.scala
+++ b/core/src/main/scala/spark/repl/SparkInterpreter.scala
diff --git a/src/scala/spark/repl/SparkInterpreterLoop.scala b/core/src/main/scala/spark/repl/SparkInterpreterLoop.scala
index 5bad0a37da..d4974009ce 100644
--- a/src/scala/spark/repl/SparkInterpreterLoop.scala
+++ b/core/src/main/scala/spark/repl/SparkInterpreterLoop.scala
@@ -129,7 +129,8 @@ extends InterpreterControl {
       settings.classpath append addedClasspath
       
     interpreter = new SparkInterpreter(settings, out) {
-      override protected def parentClassLoader = classOf[SparkInterpreterLoop].getClassLoader
+      override protected def parentClassLoader = 
+        classOf[SparkInterpreterLoop].getClassLoader
     }
     interpreter.setContextClassLoader()
     // interpreter.quietBind("settings", "spark.repl.SparkInterpreterSettings", interpreter.isettings)
diff --git a/src/scala/spark/repl/SparkInterpreterSettings.scala b/core/src/main/scala/spark/repl/SparkInterpreterSettings.scala
index ffa477785b..ffa477785b 100644
--- a/src/scala/spark/repl/SparkInterpreterSettings.scala
+++ b/core/src/main/scala/spark/repl/SparkInterpreterSettings.scala
diff --git a/src/scala/spark/repl/SparkJLineReader.scala b/core/src/main/scala/spark/repl/SparkJLineReader.scala
index 9d761c06fc..9d761c06fc 100644
--- a/src/scala/spark/repl/SparkJLineReader.scala
+++ b/core/src/main/scala/spark/repl/SparkJLineReader.scala
diff --git a/src/scala/spark/repl/SparkSimpleReader.scala b/core/src/main/scala/spark/repl/SparkSimpleReader.scala
index 2b24c4bf63..2b24c4bf63 100644
--- a/src/scala/spark/repl/SparkSimpleReader.scala
+++ b/core/src/main/scala/spark/repl/SparkSimpleReader.scala
diff --git a/src/test/spark/ParallelArraySplitSuite.scala b/core/src/test/scala/spark/ParallelArraySplitSuite.scala
index 222df4e071..222df4e071 100644
--- a/src/test/spark/ParallelArraySplitSuite.scala
+++ b/core/src/test/scala/spark/ParallelArraySplitSuite.scala
diff --git a/src/test/spark/ShuffleSuite.scala b/core/src/test/scala/spark/ShuffleSuite.scala
index a5773614e8..a5773614e8 100644
--- a/src/test/spark/ShuffleSuite.scala
+++ b/core/src/test/scala/spark/ShuffleSuite.scala
diff --git a/src/test/spark/repl/ReplSuite.scala b/core/src/test/scala/spark/repl/ReplSuite.scala
index 8b38cde85f..225e766c71 100644
--- a/src/test/spark/repl/ReplSuite.scala
+++ b/core/src/test/scala/spark/repl/ReplSuite.scala
@@ -1,6 +1,10 @@
 package spark.repl
 
 import java.io._
+import java.net.URLClassLoader
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConversions._
 
 import org.scalatest.FunSuite
 
@@ -8,9 +12,20 @@ class ReplSuite extends FunSuite {
   def runInterpreter(master: String, input: String): String = {
     val in = new BufferedReader(new StringReader(input + "\n"))
     val out = new StringWriter()
+    val cl = getClass.getClassLoader
+    var paths = new ArrayBuffer[String]
+    if (cl.isInstanceOf[URLClassLoader]) {
+      val urlLoader = cl.asInstanceOf[URLClassLoader]
+      for (url <- urlLoader.getURLs) {
+        if (url.getProtocol == "file") {
+          paths += url.getFile
+        }
+      }
+    }
     val interp = new SparkInterpreterLoop(in, new PrintWriter(out), master)
     spark.repl.Main.interp = interp
-    interp.main(new Array[String](0))
+    val separator = System.getProperty("path.separator")
+    interp.main(Array("-classpath", paths.mkString(separator)))
     spark.repl.Main.interp = null
     return out.toString
   }
diff --git a/src/examples/BroadcastTest.scala b/examples/src/main/scala/spark/examples/BroadcastTest.scala
index 1d89b3ed13..f3a173b183 100644
--- a/src/examples/BroadcastTest.scala
+++ b/examples/src/main/scala/spark/examples/BroadcastTest.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import spark.SparkContext
 
 object BroadcastTest {
diff --git a/src/examples/CpuHog.scala b/examples/src/main/scala/spark/examples/CpuHog.scala
index f37c6f7824..94b3709850 100644
--- a/src/examples/CpuHog.scala
+++ b/examples/src/main/scala/spark/examples/CpuHog.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import spark._
 
 object CpuHog {
diff --git a/src/examples/HdfsTest.scala b/examples/src/main/scala/spark/examples/HdfsTest.scala
index e678154aab..072b4ce417 100644
--- a/src/examples/HdfsTest.scala
+++ b/examples/src/main/scala/spark/examples/HdfsTest.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import spark._
 
 object HdfsTest {
diff --git a/src/examples/LocalALS.scala b/examples/src/main/scala/spark/examples/LocalALS.scala
index a976a5e1c5..10360dab3d 100644
--- a/src/examples/LocalALS.scala
+++ b/examples/src/main/scala/spark/examples/LocalALS.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import java.util.Random
 import scala.math.sqrt
 import cern.jet.math._
diff --git a/src/examples/LocalFileLR.scala b/examples/src/main/scala/spark/examples/LocalFileLR.scala
index 3d3bb60677..cc14aa7090 100644
--- a/src/examples/LocalFileLR.scala
+++ b/examples/src/main/scala/spark/examples/LocalFileLR.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import java.util.Random
 import Vector._
 
diff --git a/src/examples/LocalLR.scala b/examples/src/main/scala/spark/examples/LocalLR.scala
index 175907e551..3fd3f88fa8 100644
--- a/src/examples/LocalLR.scala
+++ b/examples/src/main/scala/spark/examples/LocalLR.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import java.util.Random
 import Vector._
 
diff --git a/src/examples/LocalPi.scala b/examples/src/main/scala/spark/examples/LocalPi.scala
index c61b3e53d4..9457472f2d 100644
--- a/src/examples/LocalPi.scala
+++ b/examples/src/main/scala/spark/examples/LocalPi.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import scala.math.random
 import spark._
 import SparkContext._
diff --git a/src/examples/SleepJob.scala b/examples/src/main/scala/spark/examples/SleepJob.scala
index a5e0ea0dc2..02673a5f88 100644
--- a/src/examples/SleepJob.scala
+++ b/examples/src/main/scala/spark/examples/SleepJob.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import spark._
 
 object SleepJob {
diff --git a/src/examples/SparkALS.scala b/examples/src/main/scala/spark/examples/SparkALS.scala
index 6fae3c0940..08e0420371 100644
--- a/src/examples/SparkALS.scala
+++ b/examples/src/main/scala/spark/examples/SparkALS.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import java.io.Serializable
 import java.util.Random
 import scala.math.sqrt
diff --git a/src/examples/SparkHdfsLR.scala b/examples/src/main/scala/spark/examples/SparkHdfsLR.scala
index f14d48b17c..4c71fd0845 100644
--- a/src/examples/SparkHdfsLR.scala
+++ b/examples/src/main/scala/spark/examples/SparkHdfsLR.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import java.util.Random
 import scala.math.exp
 import Vector._
diff --git a/src/examples/SparkLR.scala b/examples/src/main/scala/spark/examples/SparkLR.scala
index 71f9aea624..d08f5d3f01 100644
--- a/src/examples/SparkLR.scala
+++ b/examples/src/main/scala/spark/examples/SparkLR.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import java.util.Random
 import scala.math.exp
 import Vector._
diff --git a/src/examples/SparkPi.scala b/examples/src/main/scala/spark/examples/SparkPi.scala
index f055614125..31c6c5b9b1 100644
--- a/src/examples/SparkPi.scala
+++ b/examples/src/main/scala/spark/examples/SparkPi.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 import scala.math.random
 import spark._
 import SparkContext._
diff --git a/src/examples/Vector.scala b/examples/src/main/scala/spark/examples/Vector.scala
index e9fbdca752..ea70626e71 100644
--- a/src/examples/Vector.scala
+++ b/examples/src/main/scala/spark/examples/Vector.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
 @serializable class Vector(val elements: Array[Double]) {
   def length = elements.length
   
diff --git a/project/build.properties b/project/build.properties
new file mode 100644
index 0000000000..b048c01e2a
--- /dev/null
+++ b/project/build.properties
@@ -0,0 +1,8 @@
+#Project properties
+#Sat Nov 13 21:57:32 PST 2010
+project.organization=UC Berkeley
+project.name=Spark
+sbt.version=0.7.5.RC0
+project.version=0.0.0
+build.scala.versions=2.8.1
+project.initialize=false
diff --git a/project/build/SparkProject.scala b/project/build/SparkProject.scala
new file mode 100644
index 0000000000..63bd3ff578
--- /dev/null
+++ b/project/build/SparkProject.scala
@@ -0,0 +1,72 @@
+import sbt._
+import sbt.Process._
+
+import assembly._
+
+import de.element34.sbteclipsify._
+
+
+class SparkProject(info: ProjectInfo)
+extends ParentProject(info) with IdeaProject
+{
+  lazy val core = project("core", "Spark Core", new CoreProject(_))
+
+  lazy val examples =
+    project("examples", "Spark Examples", new ExamplesProject(_), core)
+
+  class CoreProject(info: ProjectInfo)
+  extends DefaultProject(info) with Eclipsify with IdeaProject with AssemblyBuilder
+  {
+    def testReportDir = outputPath / "test-report"
+
+    // Create an XML test report using ScalaTest's -u option. Unfortunately
+    // there is currently no way to call this directly from SBT without
+    // executing a subprocess.
+    lazy val testReport = task {
+      log.info("Creating " + testReportDir + "...")
+      if (!testReportDir.exists) {
+        testReportDir.asFile.mkdirs()
+      }
+
+      log.info("Executing org.scalatest.tools.Runner...")
+      val command = ("scala -classpath " + testClasspath.absString + 
+                     " org.scalatest.tools.Runner -o " + 
+                     " -u " + testReportDir.absolutePath +
+                     " -p " + (outputPath / "test-classes").absolutePath)
+      val process = Process(command, path("."), "JAVA_OPTS" -> "-Xmx500m")
+      process !
+
+      None
+    }.dependsOn(compile, testCompile).describedAs("Generate XML test report.")
+
+    def singleJarExclude(base: PathFinder) = {
+      (base / "scala" ** "*") +++ ( // exclude scala library
+        (base / "META-INF" ** "*") --- // generally ignore the hell out of META-INF
+        (base / "META-INF" / "services" ** "*") --- // include all service providers
+        (base / "META-INF" / "maven" ** "*")) // include all Maven POMs and such
+    }
+
+    def singleJarTempDir = outputPath / "single-jar-classes"
+
+    def singleJarOutputPath =
+      outputPath / (name.toLowerCase.replace(" ", "-") + "-single-jar-" + version.toString + ".jar")
+
+    // Create a JAR with Spark Core and all its dependencies. We use some methods in
+    // the AssemblyBuilder plugin, but because this plugin attempts to package Scala
+    // too, we leave that out using our own exclude filter (singleJarExclude).
+    lazy val singleJar = {
+      packageTask(
+        Path.lazyPathFinder(assemblyPaths(singleJarTempDir,
+                                          assemblyClasspath,
+                                          assemblyExtraJars,
+                                          singleJarExclude)),
+        singleJarOutputPath,
+        packageOptions)
+    }.dependsOn(compile).describedAs("Build a single JAR with project and its dependencies")
+  }
+
+  class ExamplesProject(info: ProjectInfo)
+  extends DefaultProject(info) with Eclipsify with IdeaProject
+  {
+  }
+}
diff --git a/project/plugins/SparkProjectPlugins.scala b/project/plugins/SparkProjectPlugins.scala
new file mode 100644
index 0000000000..b07dfafcfb
--- /dev/null
+++ b/project/plugins/SparkProjectPlugins.scala
@@ -0,0 +1,11 @@
+import sbt._
+
+class SparkProjectPlugins(info: ProjectInfo) extends PluginDefinition(info) {
+  val eclipse = "de.element34" % "sbt-eclipsify" % "0.7.0"
+
+  val sbtIdeaRepo = "sbt-idea-repo" at "http://mpeltonen.github.com/maven/"
+  val sbtIdea = "com.github.mpeltonen" % "sbt-idea-plugin" % "0.2.0"
+
+  val codaRepo = "Coda Hale's Repository" at "http://repo.codahale.com/"
+  val assemblySBT = "com.codahale" % "assembly-sbt" % "0.1.1"
+}
diff --git a/project/plugins/project/build.properties b/project/plugins/project/build.properties
new file mode 100644
index 0000000000..f834af277c
--- /dev/null
+++ b/project/plugins/project/build.properties
@@ -0,0 +1,3 @@
+#Project properties
+#Wed Feb 02 16:50:14 PST 2011
+plugin.uptodate=true
diff --git a/run b/run
index 5c8943c91b..c19b221ebc 100755
--- a/run
+++ b/run
@@ -25,7 +25,7 @@ fi
 
 # Set JAVA_OPTS to be able to load native libraries and to set heap size
 JAVA_OPTS="$SPARK_JAVA_OPTS"
-JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH:$FWDIR/third_party:$FWDIR/src/native:$MESOS_LIBRARY_PATH"
+JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH:$FWDIR/lib:$FWDIR/src/main/native:$MESOS_LIBRARY_PATH"
 JAVA_OPTS+=" -Xms$SPARK_MEM -Xmx$SPARK_MEM"
 # Load extra JAVA_OPTS from conf/java-opts, if it exists
 if [ -e $FWDIR/conf/java-opts ] ; then
@@ -33,23 +33,27 @@ if [ -e $FWDIR/conf/java-opts ] ; then
 fi
 export JAVA_OPTS
 
+CORE_DIR=$FWDIR/core
+EXAMPLES_DIR=$FWDIR/examples
+
 # Build up classpath
-CLASSPATH="$SPARK_CLASSPATH:$FWDIR/build/classes:$MESOS_CLASSPATH"
+CLASSPATH="$SPARK_CLASSPATH:$CORE_DIR/target/scala_2.8.1/classes:$MESOS_CLASSPATH"
 CLASSPATH+=:$FWDIR/conf
-CLASSPATH+=:$FWDIR/third_party/mesos.jar
-CLASSPATH+=:$FWDIR/third_party/asm-3.2/lib/all/asm-all-3.2.jar
-CLASSPATH+=:$FWDIR/third_party/colt.jar
-CLASSPATH+=:$FWDIR/third_party/guava-r07/guava-r07.jar
-CLASSPATH+=:$FWDIR/third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar
-CLASSPATH+=:$FWDIR/third_party/scalatest-1.2/scalatest-1.2.jar
-CLASSPATH+=:$FWDIR/third_party/scalacheck_2.8.0-1.7.jar
-CLASSPATH+=:$FWDIR/third_party/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
-CLASSPATH+=:$FWDIR/third_party/jetty-7.1.6.v20100715/servlet-api-2.5.jar
-CLASSPATH+=:$FWDIR/third_party/apache-log4j-1.2.16/log4j-1.2.16.jar
-CLASSPATH+=:$FWDIR/third_party/slf4j-1.6.1/slf4j-api-1.6.1.jar
-CLASSPATH+=:$FWDIR/third_party/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
-CLASSPATH+=:$FWDIR/third_party/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
-for jar in $FWDIR/third_party/hadoop-0.20.0/lib/*.jar; do
+CLASSPATH+=:$CORE_DIR/lib/mesos.jar
+CLASSPATH+=:$CORE_DIR/lib/asm-3.2/lib/all/asm-all-3.2.jar
+CLASSPATH+=:$CORE_DIR/lib/colt.jar
+CLASSPATH+=:$CORE_DIR/lib/guava-r07/guava-r07.jar
+CLASSPATH+=:$CORE_DIR/lib/hadoop-0.20.0/hadoop-0.20.0-core.jar
+CLASSPATH+=:$CORE_DIR/lib/scalatest-1.2/scalatest-1.2.jar
+CLASSPATH+=:$CORE_DIR/lib/scalacheck_2.8.0-1.7.jar
+CLASSPATH+=:$CORE_DIR/lib/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
+CLASSPATH+=:$CORE_DIR/lib/jetty-7.1.6.v20100715/servlet-api-2.5.jar
+CLASSPATH+=:$CORE_DIR/lib/apache-log4j-1.2.16/log4j-1.2.16.jar
+CLASSPATH+=:$CORE_DIR/lib/slf4j-1.6.1/slf4j-api-1.6.1.jar
+CLASSPATH+=:$CORE_DIR/lib/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
+CLASSPATH+=:$CORE_DIR/lib/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
+CLASSPATH+=:$EXAMPLES_DIR/target/scala_2.8.1/classes
+for jar in $CORE_DIR/lib/hadoop-0.20.0/lib/*.jar; do
   CLASSPATH+=:$jar
 done
 export CLASSPATH # Needed for spark-shell
diff --git a/sbt/sbt b/sbt/sbt
new file mode 100755
index 0000000000..87395fa28b
--- /dev/null
+++ b/sbt/sbt
@@ -0,0 +1,2 @@
+#!/bin/bash
+java -Xmx700M -jar `dirname $0`/sbt-launch-*.jar "$@"
diff --git a/sbt/sbt-launch-0.7.5.RC0.jar b/sbt/sbt-launch-0.7.5.RC0.jar
new file mode 100644
index 0000000000..e8a5c00f80
--- /dev/null
+++ b/sbt/sbt-launch-0.7.5.RC0.jar