diff options
-rw-r--r-- | core/src/main/scala/spark/api/python/PythonRDD.scala | 2 | ||||
-rw-r--r-- | docs/_plugins/copy_api_dirs.rb | 8 | ||||
-rwxr-xr-x | pyspark-shell | 3 | ||||
-rwxr-xr-x | pyspark/pyspark-shell | 3 | ||||
-rw-r--r-- | python/.gitignore (renamed from pyspark/.gitignore) | 0 | ||||
-rw-r--r-- | python/epydoc.conf (renamed from pyspark/epydoc.conf) | 0 | ||||
-rw-r--r-- | python/examples/kmeans.py (renamed from pyspark/examples/kmeans.py) | 0 | ||||
-rwxr-xr-x | python/examples/logistic_regression.py (renamed from pyspark/examples/logistic_regression.py) | 0 | ||||
-rw-r--r-- | python/examples/pi.py (renamed from pyspark/examples/pi.py) | 0 | ||||
-rw-r--r-- | python/examples/transitive_closure.py (renamed from pyspark/examples/transitive_closure.py) | 0 | ||||
-rw-r--r-- | python/examples/wordcount.py (renamed from pyspark/examples/wordcount.py) | 0 | ||||
-rw-r--r-- | python/lib/PY4J_LICENSE.txt (renamed from pyspark/lib/PY4J_LICENSE.txt) | 0 | ||||
-rw-r--r-- | python/lib/PY4J_VERSION.txt (renamed from pyspark/lib/PY4J_VERSION.txt) | 0 | ||||
-rw-r--r-- | python/lib/py4j0.7.egg (renamed from pyspark/lib/py4j0.7.egg) | bin | 191756 -> 191756 bytes | |||
-rw-r--r-- | python/lib/py4j0.7.jar (renamed from pyspark/lib/py4j0.7.jar) | bin | 103286 -> 103286 bytes | |||
-rw-r--r-- | python/pyspark/__init__.py (renamed from pyspark/pyspark/__init__.py) | 2 | ||||
-rw-r--r-- | python/pyspark/broadcast.py (renamed from pyspark/pyspark/broadcast.py) | 0 | ||||
-rw-r--r-- | python/pyspark/cloudpickle.py (renamed from pyspark/pyspark/cloudpickle.py) | 0 | ||||
-rw-r--r-- | python/pyspark/context.py (renamed from pyspark/pyspark/context.py) | 0 | ||||
-rw-r--r-- | python/pyspark/java_gateway.py (renamed from pyspark/pyspark/java_gateway.py) | 0 | ||||
-rw-r--r-- | python/pyspark/join.py (renamed from pyspark/pyspark/join.py) | 0 | ||||
-rw-r--r-- | python/pyspark/rdd.py (renamed from pyspark/pyspark/rdd.py) | 0 | ||||
-rw-r--r-- | python/pyspark/serializers.py (renamed from pyspark/pyspark/serializers.py) | 0 | ||||
-rw-r--r-- | python/pyspark/shell.py (renamed from pyspark/pyspark/shell.py) | 0 | ||||
-rw-r--r-- | python/pyspark/worker.py (renamed from pyspark/pyspark/worker.py) | 0 | ||||
-rwxr-xr-x | run | 2 | ||||
-rwxr-xr-x | run-pyspark (renamed from pyspark/run-pyspark) | 4 | ||||
-rw-r--r-- | run2.cmd | 2 |
28 files changed, 13 insertions, 13 deletions
diff --git a/core/src/main/scala/spark/api/python/PythonRDD.scala b/core/src/main/scala/spark/api/python/PythonRDD.scala index 19a039e330..cf60d14f03 100644 --- a/core/src/main/scala/spark/api/python/PythonRDD.scala +++ b/core/src/main/scala/spark/api/python/PythonRDD.scala @@ -38,7 +38,7 @@ private[spark] class PythonRDD[T: ClassManifest]( override def compute(split: Split, context: TaskContext): Iterator[Array[Byte]] = { val SPARK_HOME = new ProcessBuilder().environment().get("SPARK_HOME") - val pb = new ProcessBuilder(Seq(pythonExec, SPARK_HOME + "/pyspark/pyspark/worker.py")) + val pb = new ProcessBuilder(Seq(pythonExec, SPARK_HOME + "/python/pyspark/worker.py")) // Add the environmental variables to the process. val currentEnvVars = pb.environment() diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index 577f3ebe70..c9ce589c1b 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -30,8 +30,8 @@ if ENV['SKIP_SCALADOC'] != '1' end if ENV['SKIP_EPYDOC'] != '1' - puts "Moving to pyspark directory and building epydoc." - cd("../pyspark") + puts "Moving to python directory and building epydoc." + cd("../python") puts `epydoc --config epydoc.conf` puts "Moving back into docs dir." @@ -40,8 +40,8 @@ if ENV['SKIP_EPYDOC'] != '1' puts "echo making directory pyspark" mkdir_p "pyspark" - puts "cp -r ../pyspark/docs/. api/pyspark" - cp_r("../pyspark/docs/.", "api/pyspark") + puts "cp -r ../python/docs/. api/pyspark" + cp_r("../python/docs/.", "api/pyspark") cd("..") end diff --git a/pyspark-shell b/pyspark-shell new file mode 100755 index 0000000000..27aaac3a26 --- /dev/null +++ b/pyspark-shell @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +FWDIR="`dirname $0`" +exec $FWDIR/run-pyspark $FWDIR/python/pyspark/shell.py "$@" diff --git a/pyspark/pyspark-shell b/pyspark/pyspark-shell deleted file mode 100755 index e3736826e8..0000000000 --- a/pyspark/pyspark-shell +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -FWDIR="`dirname $0`" -exec $FWDIR/run-pyspark $FWDIR/pyspark/shell.py "$@" diff --git a/pyspark/.gitignore b/python/.gitignore index 5c56e638f9..5c56e638f9 100644 --- a/pyspark/.gitignore +++ b/python/.gitignore diff --git a/pyspark/epydoc.conf b/python/epydoc.conf index 91ac984ba2..91ac984ba2 100644 --- a/pyspark/epydoc.conf +++ b/python/epydoc.conf diff --git a/pyspark/examples/kmeans.py b/python/examples/kmeans.py index ad2be21178..ad2be21178 100644 --- a/pyspark/examples/kmeans.py +++ b/python/examples/kmeans.py diff --git a/pyspark/examples/logistic_regression.py b/python/examples/logistic_regression.py index f13698a86f..f13698a86f 100755 --- a/pyspark/examples/logistic_regression.py +++ b/python/examples/logistic_regression.py diff --git a/pyspark/examples/pi.py b/python/examples/pi.py index 127cba029b..127cba029b 100644 --- a/pyspark/examples/pi.py +++ b/python/examples/pi.py diff --git a/pyspark/examples/transitive_closure.py b/python/examples/transitive_closure.py index 73f7f8fbaf..73f7f8fbaf 100644 --- a/pyspark/examples/transitive_closure.py +++ b/python/examples/transitive_closure.py diff --git a/pyspark/examples/wordcount.py b/python/examples/wordcount.py index 857160624b..857160624b 100644 --- a/pyspark/examples/wordcount.py +++ b/python/examples/wordcount.py diff --git a/pyspark/lib/PY4J_LICENSE.txt b/python/lib/PY4J_LICENSE.txt index a70279ca14..a70279ca14 100644 --- a/pyspark/lib/PY4J_LICENSE.txt +++ b/python/lib/PY4J_LICENSE.txt diff --git a/pyspark/lib/PY4J_VERSION.txt b/python/lib/PY4J_VERSION.txt index 04a0cd52a8..04a0cd52a8 100644 --- a/pyspark/lib/PY4J_VERSION.txt +++ b/python/lib/PY4J_VERSION.txt diff --git a/pyspark/lib/py4j0.7.egg b/python/lib/py4j0.7.egg Binary files differindex f8a339d8ee..f8a339d8ee 100644 --- a/pyspark/lib/py4j0.7.egg +++ b/python/lib/py4j0.7.egg diff --git a/pyspark/lib/py4j0.7.jar b/python/lib/py4j0.7.jar Binary files differindex 73b7ddb7d1..73b7ddb7d1 100644 --- a/pyspark/lib/py4j0.7.jar +++ b/python/lib/py4j0.7.jar diff --git a/pyspark/pyspark/__init__.py b/python/pyspark/__init__.py index 1ab360a666..c595ae0842 100644 --- a/pyspark/pyspark/__init__.py +++ b/python/pyspark/__init__.py @@ -10,7 +10,7 @@ Public classes: """ import sys import os -sys.path.insert(0, os.path.join(os.environ["SPARK_HOME"], "pyspark/lib/py4j0.7.egg")) +sys.path.insert(0, os.path.join(os.environ["SPARK_HOME"], "python/lib/py4j0.7.egg")) from pyspark.context import SparkContext diff --git a/pyspark/pyspark/broadcast.py b/python/pyspark/broadcast.py index 93876fa738..93876fa738 100644 --- a/pyspark/pyspark/broadcast.py +++ b/python/pyspark/broadcast.py diff --git a/pyspark/pyspark/cloudpickle.py b/python/pyspark/cloudpickle.py index 6a7c23a069..6a7c23a069 100644 --- a/pyspark/pyspark/cloudpickle.py +++ b/python/pyspark/cloudpickle.py diff --git a/pyspark/pyspark/context.py b/python/pyspark/context.py index 6172d69dcf..6172d69dcf 100644 --- a/pyspark/pyspark/context.py +++ b/python/pyspark/context.py diff --git a/pyspark/pyspark/java_gateway.py b/python/pyspark/java_gateway.py index 2329e536cc..2329e536cc 100644 --- a/pyspark/pyspark/java_gateway.py +++ b/python/pyspark/java_gateway.py diff --git a/pyspark/pyspark/join.py b/python/pyspark/join.py index 7036c47980..7036c47980 100644 --- a/pyspark/pyspark/join.py +++ b/python/pyspark/join.py diff --git a/pyspark/pyspark/rdd.py b/python/pyspark/rdd.py index cbffb6cc1f..cbffb6cc1f 100644 --- a/pyspark/pyspark/rdd.py +++ b/python/pyspark/rdd.py diff --git a/pyspark/pyspark/serializers.py b/python/pyspark/serializers.py index 9a5151ea00..9a5151ea00 100644 --- a/pyspark/pyspark/serializers.py +++ b/python/pyspark/serializers.py diff --git a/pyspark/pyspark/shell.py b/python/pyspark/shell.py index bd39b0283f..bd39b0283f 100644 --- a/pyspark/pyspark/shell.py +++ b/python/pyspark/shell.py diff --git a/pyspark/pyspark/worker.py b/python/pyspark/worker.py index 9f6b507dbd..9f6b507dbd 100644 --- a/pyspark/pyspark/worker.py +++ b/python/pyspark/worker.py @@ -63,7 +63,7 @@ CORE_DIR="$FWDIR/core" REPL_DIR="$FWDIR/repl" EXAMPLES_DIR="$FWDIR/examples" BAGEL_DIR="$FWDIR/bagel" -PYSPARK_DIR="$FWDIR/pyspark" +PYSPARK_DIR="$FWDIR/python" # Build up classpath CLASSPATH="$SPARK_CLASSPATH" diff --git a/pyspark/run-pyspark b/run-pyspark index 4d10fbea8b..deb0d708b3 100755 --- a/pyspark/run-pyspark +++ b/run-pyspark @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Figure out where the Scala framework is installed -FWDIR="$(cd `dirname $0`; cd ../; pwd)" +FWDIR="$(cd `dirname $0`; pwd)" # Export this as SPARK_HOME export SPARK_HOME="$FWDIR" @@ -18,7 +18,7 @@ fi export PYSPARK_PYTHON # Add the PySpark classes to the Python path: -export PYTHONPATH=$SPARK_HOME/pyspark/:$PYTHONPATH +export PYTHONPATH=$SPARK_HOME/python/:$PYTHONPATH # Launch with `scala` by default: if [[ "$SPARK_LAUNCH_WITH_SCALA" != "0" ]] ; then @@ -34,7 +34,7 @@ set CORE_DIR=%FWDIR%core set REPL_DIR=%FWDIR%repl set EXAMPLES_DIR=%FWDIR%examples set BAGEL_DIR=%FWDIR%bagel -set PYSPARK_DIR=%FWDIR%pyspark +set PYSPARK_DIR=%FWDIR%python rem Build up classpath set CLASSPATH=%SPARK_CLASSPATH%;%MESOS_CLASSPATH%;%FWDIR%conf;%CORE_DIR%\target\scala-%SCALA_VERSION%\classes |