From 965c82d8c4b7f2d4dfbc45ec4d47d6b6588094c3 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 25 Jan 2017 14:43:39 -0800 Subject: [SPARK-19064][PYSPARK] Fix pip installing of sub components ## What changes were proposed in this pull request? Fix instalation of mllib and ml sub components, and more eagerly cleanup cache files during test script & make-distribution. ## How was this patch tested? Updated sanity test script to import mllib and ml sub-components. Author: Holden Karau Closes #16465 from holdenk/SPARK-19064-fix-pip-install-sub-components. --- dev/run-pip-tests | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'dev/run-pip-tests') diff --git a/dev/run-pip-tests b/dev/run-pip-tests index e1da18e60b..af1b1feb70 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -78,11 +78,14 @@ for python in "${PYTHON_EXECS[@]}"; do mkdir -p "$VIRTUALENV_PATH" virtualenv --python=$python "$VIRTUALENV_PATH" source "$VIRTUALENV_PATH"/bin/activate - # Upgrade pip - pip install --upgrade pip + # Upgrade pip & friends + pip install --upgrade pip pypandoc wheel + pip install numpy # Needed so we can verify mllib imports echo "Creating pip installable source dist" cd "$FWDIR"/python + # Delete the egg info file if it exists, this can cache the setup file. + rm -rf pyspark.egg-info || echo "No existing egg info file, skipping deletion" $python setup.py sdist -- cgit v1.2.3