diff options
author | Patrick Wendell <pwendell@gmail.com> | 2014-06-10 13:13:17 -0700 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2014-06-10 13:13:33 -0700 |
commit | 1d9b7651e42a33dd27d8c7a470f12b7cb6c14385 (patch) | |
tree | 45886c2c057ea4cd09a860cc7cf16b1516ef44e4 | |
parent | 35894af8cc66056352a1e4c7ebff1b6ecb12b7b9 (diff) | |
download | spark-1d9b7651e42a33dd27d8c7a470f12b7cb6c14385.tar.gz spark-1d9b7651e42a33dd27d8c7a470f12b7cb6c14385.tar.bz2 spark-1d9b7651e42a33dd27d8c7a470f12b7cb6c14385.zip |
HOTFIX: Fix Python tests on Jenkins.
Author: Patrick Wendell <pwendell@gmail.com>
Closes #1036 from pwendell/jenkins-test and squashes the following commits:
9c99856 [Patrick Wendell] Better output during tests
71e7b74 [Patrick Wendell] Removing incorrect python path
74984db [Patrick Wendell] HOTFIX: Allow PySpark tests to run on Jenkins.
(cherry picked from commit fb499be1ac935b6f91046ec8ff23ac1267c82342)
Signed-off-by: Patrick Wendell <pwendell@gmail.com>
-rwxr-xr-x | bin/pyspark | 6 | ||||
-rwxr-xr-x | dev/run-tests | 3 | ||||
-rwxr-xr-x | python/run-tests | 11 |
3 files changed, 12 insertions, 8 deletions
diff --git a/bin/pyspark b/bin/pyspark index d0fa56f319..114cbbc3a8 100755 --- a/bin/pyspark +++ b/bin/pyspark @@ -86,6 +86,10 @@ else if [[ "$IPYTHON" = "1" ]]; then exec ipython $IPYTHON_OPTS else - exec "$PYSPARK_PYTHON" + if [[ -n $SPARK_TESTING ]]; then + exec "$PYSPARK_PYTHON" -m doctest + else + exec "$PYSPARK_PYTHON" + fi fi fi diff --git a/dev/run-tests b/dev/run-tests index 6043f859ae..26df79b9c0 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -73,9 +73,6 @@ fi echo "=========================================================================" echo "Running PySpark tests" echo "=========================================================================" -if [ -z "$PYSPARK_PYTHON" ]; then - export PYSPARK_PYTHON=/usr/local/bin/python2.7 -fi ./python/run-tests echo "=========================================================================" diff --git a/python/run-tests b/python/run-tests index 36a96121cb..3b4501178c 100755 --- a/python/run-tests +++ b/python/run-tests @@ -32,7 +32,8 @@ rm -f unit-tests.log rm -rf metastore warehouse function run_test() { - SPARK_TESTING=0 $FWDIR/bin/pyspark $1 2>&1 | tee -a > unit-tests.log + echo "Running test: $1" + SPARK_TESTING=1 $FWDIR/bin/pyspark $1 2>&1 | tee -a > unit-tests.log FAILED=$((PIPESTATUS[0]||$FAILED)) # Fail and exit on the first test failure. @@ -46,15 +47,17 @@ function run_test() { } +echo "Running PySpark tests. Output is in python/unit-tests.log." + run_test "pyspark/rdd.py" run_test "pyspark/context.py" run_test "pyspark/conf.py" if [ -n "$_RUN_SQL_TESTS" ]; then run_test "pyspark/sql.py" fi -run_test "-m doctest pyspark/broadcast.py" -run_test "-m doctest pyspark/accumulators.py" -run_test "-m doctest pyspark/serializers.py" +run_test "pyspark/broadcast.py" +run_test "pyspark/accumulators.py" +run_test "pyspark/serializers.py" run_test "pyspark/tests.py" run_test "pyspark/mllib/_common.py" run_test "pyspark/mllib/classification.py" |