From d3180c25d8cf0899a7238e7d24b35c5ae918cc1d Mon Sep 17 00:00:00 2001 From: Brennon York Date: Sun, 18 Oct 2015 22:45:14 -0700 Subject: [SPARK-7018][BUILD] Refactor dev/run-tests-jenkins into Python This commit refactors the `run-tests-jenkins` script into Python. This refactoring was done by brennonyork in #7401; this PR contains a few minor edits from joshrosen in order to bring it up to date with other recent changes. From the original PR description (by brennonyork): Currently a few things are left out that, could and I think should, be smaller JIRA's after this. 1. There are still a few areas where we use environment variables where we don't need to (like `CURRENT_BLOCK`). I might get around to fixing this one in lieu of everything else, but wanted to point that out. 2. The PR tests are still written in bash. I opted to not change those and just rewrite the runner into Python. This is a great follow-on JIRA IMO. 3. All of the linting scripts are still in bash as well and would likely do to just add those in as follow-on JIRA's as well. Closes #7401. Author: Brennon York Closes #9161 from JoshRosen/run-tests-jenkins-refactoring. --- python/run-tests.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) (limited to 'python') diff --git a/python/run-tests.py b/python/run-tests.py index 152f5cc98d..f5857f8c62 100755 --- a/python/run-tests.py +++ b/python/run-tests.py @@ -31,23 +31,6 @@ if sys.version < '3': import Queue else: import queue as Queue -if sys.version_info >= (2, 7): - subprocess_check_output = subprocess.check_output -else: - # SPARK-8763 - # backported from subprocess module in Python 2.7 - def subprocess_check_output(*popenargs, **kwargs): - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd, output=output) - return output # Append `SPARK_HOME/dev` to the Python path so that we can import the sparktestsupport module @@ -55,7 +38,7 @@ sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../de from sparktestsupport import SPARK_HOME # noqa (suppress pep8 warnings) -from sparktestsupport.shellutils import which # noqa +from sparktestsupport.shellutils import which, subprocess_check_output # noqa from sparktestsupport.modules import all_modules # noqa -- cgit v1.2.3